MonetDB: iot - Finished stream creation through webserver with M...

[Pedro Ferreira]

Changeset: 4b3ab86b53d0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4b3ab86b53d0
Modified Files:
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/main.py
clients/iotclient/tests/datatypesinsertstests.py
Branch: iot
Log Message:

Finished stream creation through webserver with MonetDB, now only the stream 
polling is remaining


diffs (truncated from 954 to 300 lines):

diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -65,7 +65,7 @@ def main():
 help='Baskets location directory (default: %s)' % 
DEFAULT_FILESYSTEM, metavar='DIRECTORY')
 parser.add_argument('-l', '--log', type=check_path, nargs='?', 
default=DEFAULT_LOGGING,
 help='Logging file location (default: %s)' % 
DEFAULT_LOGGING, metavar='FILE_PATH')
-parser.add_argument('-po', '--polling', type=check_positive_int, 
nargs='?', default=60,
+parser.add_argument('-po', '--polling', type=check_positive_int, 
nargs='?', default=60, metavar='POLLING',
 help='Polling interval in seconds to the database for 
streams updates (default: 60)')
 parser.add_argument('-sh', '--shost', type=check_ipv4_address, nargs='?', 
default='0.0.0.0',
 help='Web API server host (default: 0.0.0.0)', 
metavar='HOST')
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -32,19 +32,19 @@ Set the filesystem directory where the b
 
 Location of logfile. On the logfile is reported when streams are created or 
removed, when tuples are inserted and when the baskets are flushed. By default 
in UNIX systems is :code:`/var/log/iot/iotserver.log`, while on Windows is the 
:code:`iotserver.log` on the directory where the :code:`main.py` script was 
called.
 
-Host Identifier

+Web Server Behavior
+---
 
-If the *identifier* parameter is provided, an extra column on streams will be 
added with a custom name of the host for later identification.
-
-**-i  - -identifier**
-
-Use a host identifier for every new stream.
+If a stream is created with a hostname, an extra column on streams will be 
added with a custom name of the host for later identification.
 
 **-n  - -name**
 
 Host identifier name. By default is the host's MAC address.
 
+**-po  - -polling**
+
+Set the polling interval in seconds to MonetDB database for streams updates. 
By default is :code:`60` seconds.
+
 Web Server Listening
 
 
@@ -69,7 +69,7 @@ Listening port of the administration ser
 Database Connection
 ---
 
-Credentials for the MAPI connection to MonetDB database.
+Credentials for the MAPI connection to the MonetDB database.
 
 .. note:: The user's password will be prompted during the initialization of 
the server.
 
diff --git a/clients/iotclient/documentation/restful_resources.rst 
b/clients/iotclient/documentation/restful_resources.rst
--- a/clients/iotclient/documentation/restful_resources.rst
+++ b/clients/iotclient/documentation/restful_resources.rst
@@ -20,13 +20,14 @@ The administration server provides resou
 
 **GET**
 
-Returns a JSON file with details about all the streams currently created on 
the webserver. For each stream besides its schema and name, it provides the 
currently number of tuples inserted on the baskets per column, description of 
columns (`See data types `__), the flushing 
method (`See streams creation for details 
`__). An example is shown bellow:
+Returns a JSON file with details about all the streams currently created on 
the webserver. For each stream besides its schema, name and if has the hostname 
column, it provides the currently number of tuples inserted on the baskets per 
column, description of columns (`See data types 
`__), the flushing method (`See streams 
creation for details `__). An example 
is shown bellow:
 
 .. code-block:: json
 
[
  {
"tuples_inserted_per_basket": 1,
+   "hostname": false,
"columns": [
  

MonetDB: iot - Added check for hugeint type, arranged time types...

[Pedro Ferreira]

Changeset: 5e7861106555 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5e7861106555
Modified Files:
clients/iotapi/src/Streams/streampolling.py
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/main.py
sql/backends/monet5/iot/50_iot.sql
Branch: iot
Log Message:

Added check for hugeint type, arranged time types, starting generating SQL 
statements for new streams


diffs (truncated from 1734 to 300 lines):

diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -14,7 +14,7 @@ SWITCHER = [{'types': ['clob', 'url'], '
 {'types': ['decimal'], 'class': 'DecimalType'},
 {'types': ['boolean'], 'class': 'BooleanType'},
 {'types': ['date'], 'class': 'DateType'},
-{'types': ['time', 'timez'], 'class': 'TimeType'},
+{'types': ['time', 'timetz'], 'class': 'TimeType'},
 {'types': ['timestamp', 'timestamptz'], 'class': 'TimestampType'},
 {'types': ['inet'], 'class': 'INetType'},
 {'types': ['uuid'], 'class': 'UUIDType'}]
diff --git a/clients/iotclient/documentation/restful_resources.rst 
b/clients/iotclient/documentation/restful_resources.rst
--- a/clients/iotclient/documentation/restful_resources.rst
+++ b/clients/iotclient/documentation/restful_resources.rst
@@ -28,17 +28,16 @@ Returns a JSON file with details about a
  {
"tuples_inserted_per_basket": 1,
"columns": [
- {
-   "type": "real",
-   "name": "temperature",
-   "nullable": false
- },
- {
-   "type": "text",
-   "name": "sensorid",
-   "nullable": false
- }
-   ],
+{
+   "type": "real",
+   "name": "temperature",
+   "nullable": false
+},
+{
+   "type": "text",
+   "name": "sensorid",
+   "nullable": false
+}],
"flushing": {
  "base": "tuple",
  "number": 50
@@ -53,7 +52,7 @@ Returns a JSON file with details about a
 
 **POST**
 
-Creates a stream using a pre-defined JSON schema. The JSON must include the 
stream's schema, the stream's name, the stream's columns and the flushing 
method. The flushing can be either time based, tuple based or automatic 
(:code:`auto`). For tuple based flushing, the number of tuples to flush must be 
provided using the :code:`number` field. In time based flushing, the 
:code:`interval` field tells the time units between flushes and the 
:code:`unit` field must be "s", "m" or "h" for seconds, minutes or hours 
respectively. In automatic flushing, the baskets are flushed whenver a new 
batch is inserted. For columns `see data types for details 
`__.
+Creates a stream using a pre-defined JSON schema. The JSON must include the 
stream's schema, the stream's name, the stream's columns and the flushing 
method. The flushing can be either time based, tuple based or automatic 
(:code:`auto`). For tuple based flushing, the number of tuples to flush must be 
provided using the :code:`interval` field. In time based flushing, the 
:code:`interval` field tells the time units between flushes and the 
:code:`unit` field must be "s", "m" or "h" for seconds, minutes or hours 
respectively. In automatic flushing, the baskets are flushed whenever a new 
batch is inserted. For columns `see data types for details 
`__.
 
 Bellow is the JSON used to create the stream in streams_:
 
@@ -64,8 +63,8 @@ Bellow is the JSON used to create the st
  "stream": "temperature",
  "flushing": {
"base": "tuple",
-   "number": 50
-   },
+   "interval": 50
+  },
  "columns": [
{
  "type": "real",
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -1,31 +1,23 @@
 import datetime
 import json
 import pytz
-import sys
 
 from flask import request
 from flask_restful import Resource
 from jsonschema import Draft4Validator, FormatChecker
 

MonetDB: iot - Added stream polling

[Pedro Ferreira]

Changeset: c44d0bb79a54 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c44d0bb79a54
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Added stream polling


diffs (truncated from 705 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -4,7 +4,7 @@ jsonschema==2.5.1
 pymonetdb==0.1.1
 python-dateutil==2.5.3
 pytz==2016.4
-Sphinx==1.4.1
+Sphinx==1.4.3
 sphinx-rtd-theme==0.1.9
 tzlocal==1.2.2
 watchdog==0.8.3
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -328,14 +328,6 @@ class TimeType(StreamDataType):  # Store
.isoformat())
 return results
 
-def to_json_representation(self):
-json_value = super(TimeType, self).to_json_representation()
-if self._data_type == 'timez':
-json_value['with_timezone'] = True
-else:
-json_value['with_timezone'] = False
-return json_value
-
 
 class TimestampType(StreamDataType):  # it's represented with the two integers 
from time and date
 """Covers: TIMESTAMP"""
@@ -363,11 +355,3 @@ class TimestampType(StreamDataType):  # 
 results.append(datetime.combine(read_date, time(hour=hour, 
minute=minute, second=second,
 
microsecond=milliseconds * 1000)).isoformat())
 return results
-
-def to_json_representation(self):
-json_value = super(TimestampType, self).to_json_representation()
-if self._data_type == 'timestamptz':
-json_value['with_timezone'] = True
-else:
-json_value['with_timezone'] = False
-return json_value
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -1,5 +1,4 @@
 from itertools import groupby
-
 from Settings.mapiconnection import fetch_streams
 from Utilities.customthreading import PeriodicalThread
 from datatypes import *
diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,13 +1,13 @@
-python-dateutil==2.5.3
 fake-factory==0.5.7
 Flask-RESTful==0.3.5
 IPy==0.83
 iso8601==0.1.11
 jsonschema==2.5.1
 pymonetdb==0.1.1
+python-dateutil==2.5.3
 pytz==2016.4
 rfc3987==1.3.6
-Sphinx==1.4.1
+Sphinx==1.4.3
 sphinx-rtd-theme==0.1.9
 strict-rfc3339==0.7
 tzlocal==1.2.2
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -49,22 +49,21 @@ def mapi_get_webserver_streams():
 try:
 Connection.execute("START TRANSACTION")
 cursor = Connection.cursor()
-sql_string = """SELECT tables."id", tables."name", schemas."name" AS 
schema, tables."name" AS table,
-extras."has_hostname", extras."flushing", extras."unit", 
extras."interval" FROM (SELECT "id", "name",
-"schema_id" FROM sys.tables WHERE type=4) AS tables INNER JOIN 
(SELECT "id", "name" FROM sys.schemas)
-AS schemas ON (tables."schema_id"=schemas."id") LEFT JOIN (SELECT 
"table_id", "has_hostname", "flushing",
-"unit", "interval" FROM iot.webserverstreams) AS extras ON 
(tables."id"=extras."table_id")"""\
-.replace('\n', ' ')
+sql_string = """SELECT tables."id", schemas."name" AS schema, 
tables."name" AS table, extras."has_hostname",
+extras."base", extras."interval", extras."unit" FROM (SELECT "id", 
"name", "schema_id" FROM sys.tables
+WHERE type=4) AS tables INNER JOIN (SELECT "id", "name" FROM 
sys.schemas) AS schemas ON
+(tables."schema_id"=schemas."id") LEFT JOIN (SELECT "table_id", 
"has_hostname", "base", "interval", "unit"
+FROM iot.webserverstreams) AS extras ON 
(tables."id"=extras."table_id")""".replace('\n', ' ')
 cursor.execute(sql_string)
 tables = cursor.fetchall()
 
 cursor = Connection.cursor()
-sql_string = """SELECT columns."table_id", columns."name" AS column, 

MonetDB: iot - Merge changes

[Pedro Ferreira]

Changeset: d40d60cb293a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d40d60cb293a
Modified Files:
sql/backends/monet5/iot/50_iot.sql
Branch: iot
Log Message:

Merge changes


diffs (13 lines):

diff --git a/sql/backends/monet5/iot/50_iot.sql 
b/sql/backends/monet5/iot/50_iot.sql
--- a/sql/backends/monet5/iot/50_iot.sql
+++ b/sql/backends/monet5/iot/50_iot.sql
@@ -106,7 +106,7 @@ create function iot.outputplaces()
 
 -- tables for iotwebserver
 
-CREATE TABLE iot.webserverflushing (table_id INTEGER, base TINYINT, "interval" 
INTEGER NULL, unit TINYINT NULL);
+CREATE TABLE iot.webserverstreams (table_id INTEGER, has_hostname BOOLEAN, 
base TINYINT, "interval" INTEGER NULL, unit TINYINT NULL);
 
-CREATE TABLE iot.webservervalidation (column_id INTEGER, special TINYINT NULL, 
validation1 string NULL, validation2 string NULL);
+CREATE TABLE iot.webservercolumns (column_id INTEGER, special TINYINT NULL, 
validation1 STRING NULL, validation2 STRING NULL);
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Merge changes

[Pedro Ferreira]

Changeset: 3cc8124c0247 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3cc8124c0247
Modified Files:
sql/backends/monet5/iot/basket.c
sql/backends/monet5/iot/basket.h
sql/backends/monet5/iot/petrinet.c
Branch: iot
Log Message:

Merge changes


diffs (94 lines):

diff --git a/sql/backends/monet5/iot/basket.c b/sql/backends/monet5/iot/basket.c
--- a/sql/backends/monet5/iot/basket.c
+++ b/sql/backends/monet5/iot/basket.c
@@ -35,7 +35,7 @@
 
 #define _DEBUG_BASKET_ if(0)
 
-str statusname[4] = { "", "active", "paused", "locked" };
+str statusname[4] = { "", "available", "wait", "locked" };
 
 BasketRec *baskets;   /* the global iot catalog */
 static int bsktTop = 0, bsktLimit = 0;
@@ -109,7 +109,7 @@ BSKTnewbasket(sql_schema *s, sql_table *
baskets[idx].table_name = GDKstrdup(t->base.name);
baskets[idx].seen = * timestamp_nil;
 
-   baskets[idx].status = BSKTPAUSED;
+   baskets[idx].status = BSKTWAIT;
baskets[idx].count = 0;
for (o = t->columns.set->h; o; o = o->next){
 sql_column *col = o->data;
@@ -205,7 +205,7 @@ BSKTactivate(Client cntxt, MalBlkPtr mb,
idx = BSKTlocate(sch, tbl);
if( idx == 0)
throw(SQL,"basket.activate","Stream table %s.%s not 
accessible to activate\n",sch,tbl);
-   if( baskets[idx].status == BSKTPAUSED){
+   if( baskets[idx].status == BSKTWAIT){
MT_lock_set();
baskets[idx].status = BSKTAVAILABLE;
MT_lock_unset();
@@ -237,13 +237,13 @@ BSKTdeactivate(Client cntxt, MalBlkPtr m
throw(SQL,"basket.activate","Stream table %s.%s not 
accessible to deactivate\n",sch,tbl);
if( baskets[idx].status == BSKTAVAILABLE ){
MT_lock_set();
-   baskets[idx].status = BSKTPAUSED;
+   baskets[idx].status = BSKTWAIT;
MT_lock_unset();
}
} else {
MT_lock_set();
for( idx =1; idx 

MonetDB: iot - First version of web api working. Updated some do...

[Pedro Ferreira]

Changeset: 86b955aa08b7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=86b955aa08b7
Modified Files:
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Streams/streams.py
Branch: iot
Log Message:

First version of web api working. Updated some documentation. Fixed small 
errors.


diffs (230 lines):

diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -119,12 +119,17 @@ class UUIDType(StreamDataType):
 iterator = iter(array)
 
 for _ in xrange(limit):
-next_uuid = ''.join(map(lambda x: "%02x" % x, [next(iterator) for 
_ in xrange(16)]))
-next_uuid = ''.join([next_uuid[:8], '-', next_uuid[8:12], '-', 
next_uuid[12:16], '-', next_uuid[16:20],
- '-', next_uuid[20:]])
-if next_uuid == NIL_UUID:
-next_uuid = None
-results.append(next_uuid)
+next_uuid = []
+for i in xrange(20):
+if i in (4, 7, 10, 13):
+next_uuid.append("-")
+else:
+next_uuid.append("%02x" % next(iterator))
+
+built_uuid = ''.join(next_uuid)
+if built_uuid == NIL_UUID:
+built_uuid = None
+results.append(built_uuid)
 return results
 
 
@@ -178,7 +183,7 @@ class HugeIntegerType(StreamDataType):
 results = []
 iterator = iter(array)  # has to iterate two values at once, so use 
iterator
 for value in iterator:
-next_huge = value + (next(iterator) << 64)
+next_huge = next(iterator) + (value << 64)
 if next_huge == self._nullable_constant:
 results.append(None)
 else:
@@ -229,7 +234,7 @@ class DecimalType(StreamDataType):
 results = []
 iterator = iter(array)  # has to iterate two values at once, so 
use iterator
 for value in iterator:
-next_huge_decimal = value + (next(iterator) << 64)
+next_huge_decimal = next(iterator) + (value << 64)
 if next_huge_decimal == self._nullable_constant:
 results.append(None)
 else:
@@ -302,8 +307,8 @@ class TimestampType(StreamDataType):  # 
 if value == INT32_MIN and second_value == 0:
 results.append(None)
 else:  # dates in python start on year 1, so we must subtract one 
year
-read_date = date.fromordinal(value) - relativedelta(years=1)
-div1, milliseconds = divmod(second_value, 1000)
+read_date = date.fromordinal(second_value) - 
relativedelta(years=1)
+div1, milliseconds = divmod(value, 1000)
 div2, second = divmod(div1, 60)
 hour, minute = divmod(div2, 60)
 results.append(datetime.combine(read_date, time(hour=hour, 
minute=minute, second=second,
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -40,9 +40,9 @@ def stream_polling():
 if elem[3] in entry['types']:
 reflection_class = globals()[entry['class']]  # import 
everything from datatypes!!!
 new_column = reflection_class(**{'name': elem[2], 
'type': elem[3], 'typewidth': elem[4]})
-columns[elem[2]] = new_column
-new_streams[key] = IOTStream(elem[0], elem[1], 
**columns)
-break
+columns[elem[2]] = new_column  # add new column to the 
dictionary
+break
+new_streams[key] = IOTStream(schema_name=elem[0], 
stream_name=elem[1], columns=columns)
 else:
 retained_streams.append(key)
 
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -5,7 +5,7 @@ from datatypes import LITTLE_ENDIAN_ALIG
 from Settings.filesystem import get_baskets_base_location
 from Utilities.readwritelock import RWLock
 from WebSockets.websockets import notify_stream_inserts_to_clients
-from watchdog.events import FileSystemEventHandler
+from watchdog.events import FileSystemEventHandler, DirCreatedEvent, 

MonetDB: iot - Small typos correction

[Pedro Ferreira]

Changeset: 81a1c44abe91 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=81a1c44abe91
Modified Files:
clients/iotapi/documentation/websockets_api.rst
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Streams/streamscontext.py
Branch: iot
Log Message:

Small typos correction


diffs (95 lines):

diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -15,8 +15,8 @@ The client must always provide a JSON st
 
 The following sections explain the available :code:`request` fields.
 
-sub

+sub/subscribe
+-
 
 Subscribes for new basket creations from a specific stream. Whenever a basket 
is created, the server sends a notification message indicating the number of 
inserted tuples in the new basket. The user has to specify the stream's name 
and schema. To subscribe to a temperature stream, the following would suffice:
 
@@ -28,8 +28,8 @@ Subscribes for new basket creations from
 "stream": "temperature"
 }
 
-unsub
--
+unsub/unsubscribe
+-
 
 Unsubscribes a previous subscribed stream for a client. The user has to 
specify the stream's name and schema. The example is the same as above, just 
changing the request keyword.
 
@@ -86,7 +86,7 @@ An internal error occurred in the server
 
 {
 "response": "error",
-"message": "Stream measures.temperature not present in the user's 
subscriptions!"
+"message": "Stream measures.temperature is not present in the user's 
subscriptions!"
 }
 
 subscribed
@@ -146,7 +146,7 @@ Notification of a new basket creation fr
 read
 
 
-Response message for a read query. Contains the reconstructed tuples 
listening. The tuples are listened in the say they are inserted in the RESTful 
webserver. If a column has a null value, the JSON's :code:`null` value will be 
listened. A possible query result for the above stream:
+Response message for a read query. Contains the reconstructed tuples 
listening. The tuples are listened in the say they are inserted in the RESTful 
Webserver. If a column has a null value, the JSON's :code:`null` value will be 
listened. A possible query result for the above stream:
 
 .. code-block:: json
 
@@ -174,7 +174,7 @@ Response message for a read query. Conta
 info
 
 
-Message with details about a stream including both columns and baskets 
details. Note that the possible types list are restricted to the MonetDB 
kernel.  A possible example for the above stream:
+Message with details about a stream including both columns and baskets 
details. Note that the possible types list are restricted to the MonetDB 
kernel. A possible example for the above stream:
 
 .. code-block:: json
 
@@ -212,7 +212,7 @@ Message with details about a stream incl
 data
 
 
-Return a info messaging regarding all the streams in the system. An example 
with the temperatures stream:
+Returns a info message regarding all the streams in the system. An example 
with the temperatures stream:
 
 .. code-block:: json
 
diff --git a/clients/iotapi/src/WebSockets/websockets.py 
b/clients/iotapi/src/WebSockets/websockets.py
--- a/clients/iotapi/src/WebSockets/websockets.py
+++ b/clients/iotapi/src/WebSockets/websockets.py
@@ -69,7 +69,7 @@ class IOTAPI(WebSocket):
 elif input_schema['request'] in UNSUBSCRIBE_OPTS:
 self.unsubscribe(input_schema['schema'], 
input_schema['stream'])
 elif input_schema['request'] in READ_OPTS:
-
self.read_stream_batch(input_schema['schema'],input_schema['stream'], 
int(input_schema['basket']),
+self.read_stream_batch(input_schema['schema'], 
input_schema['stream'], int(input_schema['basket']),
int(input_schema['limit']), 
int(input_schema['offset']))
 elif input_schema['request'] in INFO_OPTS:
 if len(input_schema) == 1:  # get all streams information
@@ -95,7 +95,7 @@ class IOTAPI(WebSocket):
 if concatenated_name not in self._subscriptions:
 self._subscriptions_locker.release()
 self.sendJSONMessage(response="error", message={"message": "Stream 
" + concatenated_name +
-   " not 
present in the user's subscriptions!"})
+   " is 
not present in the user's subscriptions!"})
 else:
 del self._subscriptions[concatenated_name]
 self._subscriptions_locker.release()
diff --git a/clients/iotclient/src/Streams/streamscontext.py 
b/clients/iotclient/src/Streams/streamscontext.py
--- a/clients/iotclient/src/Streams/streamscontext.py
+++ b/clients/iotclient/src/Streams/streamscontext.py
@@ -102,7 +102,7 @@ class 

MonetDB: iot - Added stream context merge and type checking whil...

[Pedro Ferreira]

Changeset: d2138293ca3a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d2138293ca3a
Modified Files:
clients/iotapi/src/Streams/streampolling.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Added stream context merge and type checking while reading created streams from 
the database


diffs (truncated from 565 to 300 lines):

diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -27,10 +27,10 @@ def init_stream_polling_thread(interval)
 # elem[0] is schema. elem[1] is name, elem[2] is column name, elem[3] is type, 
elem[4] is type_digits
 # elem[5] is type_scale elem[6] is default value elem[7] is nullable
 def stream_polling():
-current_streams = Streams_Context.get_existing_streams()
+array = fetch_streams()  # TODO check whenever stream's columns are updated
 retained_streams = []
 new_streams = {}
-array = fetch_streams()  # TODO check whenever stream's columns are updated
+current_streams = Streams_Context.get_existing_streams()
 
 if array is not None:
 for key, group in groupby(array, lambda x: 
Streams_Context.get_context_entry_name(x[0], x[1])):
@@ -48,5 +48,4 @@ def stream_polling():
 else:
 retained_streams.append(key)
 
-retained_streams_final = [key for key in current_streams if key in 
retained_streams]
-Streams_Context.merge_context(retained_streams_final, new_streams)
+Streams_Context.merge_context(retained_streams, new_streams)
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -1,5 +1,5 @@
+import pymonetdb
 import sys
-import pymonetdb
 
 from Settings.iotlogger import add_log
 from Streams.streamscontext import IOTStreams
@@ -49,11 +49,11 @@ def mapi_get_webserver_streams():
 try:
 Connection.execute("START TRANSACTION")
 cursor = Connection.cursor()
-sql_string = """SELECT tables."id", schemas."name" AS schema, 
tables."name" AS table, extras."has_hostname",
-extras."base", extras."interval", extras."unit" FROM (SELECT "id", 
"name", "schema_id" FROM sys.tables
-WHERE type=4) AS tables INNER JOIN (SELECT "id", "name" FROM 
sys.schemas) AS schemas ON
-(tables."schema_id"=schemas."id") LEFT JOIN (SELECT "table_id", 
"has_hostname", "base", "interval", "unit"
-FROM iot.webserverstreams) AS extras ON 
(tables."id"=extras."table_id")""".replace('\n', ' ')
+sql_string = """SELECT tables."id", schemas."name" AS schema, 
tables."name" AS table, extras."base",
+extras."interval", extras."unit" FROM (SELECT "id", "name", 
"schema_id" FROM sys.tables WHERE type=4)
+AS tables INNER JOIN (SELECT "id", "name" FROM sys.schemas) AS 
schemas ON (tables."schema_id"=schemas."id")
+LEFT JOIN (SELECT "table_id", "has_hostname", "base", "interval", 
"unit" FROM iot.webserverstreams)
+AS extras ON (tables."id"=extras."table_id")""".replace('\n', ' ')
 cursor.execute(sql_string)
 tables = cursor.fetchall()
 
@@ -72,7 +72,7 @@ def mapi_get_webserver_streams():
 return tables, columns
 except BaseException as ex:
 add_log(50, ex)
-return [], []
+raise
 
 
 def mapi_create_stream(stream):
@@ -98,7 +98,7 @@ def mapi_create_stream(stream):
 "'"]))  # get the created table id
 table_id = str(cursor.fetchall()[0][0])
 cursor.execute(''.join(["INSERT INTO iot.webserverstreams VALUES (", 
table_id, flush_statement, ")"]))
-cursor.execute('SELECT id, "name" FROM sys.columns WHERE table_id=' + 
table_id)
+cursor.execute("SELECT id, \"name\" FROM sys.columns WHERE table_id=" 
+ table_id)
 columns = cursor.fetchall()
 
 inserts = []
@@ -114,6 +114,7 @@ def mapi_create_stream(stream):
 stream.set_delete_ids(table_id, colums_ids)
 except BaseException as ex:
 add_log(50, ex)
+raise
 
 
 def mapi_delete_stream(schema, stream, stream_id, columns_ids):
@@ -125,6 +126,7 @@ def mapi_delete_stream(schema, stream, s
 Connection.commit()
 except BaseException as ex:
 add_log(50, ex)
+raise
 
 
 def mapi_flush_baskets(schema, stream, baskets):
diff --git a/clients/iotclient/src/Streams/streampolling.py 
b/clients/iotclient/src/Streams/streampolling.py
--- a/clients/iotclient/src/Streams/streampolling.py
+++ 

MonetDB: iot - Updated create table statement

[Pedro Ferreira]

Changeset: 282ddcc5d5c5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=282ddcc5d5c5
Modified Files:
sql/backends/monet5/iot/50_iot.sql
Branch: iot
Log Message:

Updated create table statement


diffs (12 lines):

diff --git a/sql/backends/monet5/iot/50_iot.sql 
b/sql/backends/monet5/iot/50_iot.sql
--- a/sql/backends/monet5/iot/50_iot.sql
+++ b/sql/backends/monet5/iot/50_iot.sql
@@ -106,7 +106,7 @@ create function iot.outputplaces()
 
 -- tables for iotwebserver
 
-CREATE TABLE iot.webserverstreams (table_id INTEGER, has_hostname BOOLEAN, 
base TINYINT, "interval" INTEGER NULL, unit TINYINT NULL);
+CREATE TABLE iot.webserverstreams (table_id INTEGER, base TINYINT, "interval" 
INTEGER NULL, unit TINYINT NULL);
 
 CREATE TABLE iot.webservercolumns (column_id INTEGER, special TINYINT NULL, 
validation1 STRING NULL, validation2 STRING NULL);
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Fixed imports, created unique mapi connections fo...

[Pedro Ferreira]

Changeset: 6c4407fe078d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6c4407fe078d
Modified Files:
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotapi/src/Utilities/readwritelock.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotapi/src/main.py
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/customthreading.py
clients/iotclient/src/Utilities/readwritelock.py
clients/iotclient/src/main.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Fixed imports, created unique mapi connections for each stream, fixed small bugs


diffs (truncated from 1302 to 300 lines):

diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -1,7 +1,7 @@
 import os
 import sys
 
-from iotlogger import add_log
+from .iotlogger import add_log
 
 Baskets_Location = None
 
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -1,7 +1,7 @@
 import sys
 
-import pymonetdb
-from Settings.iotlogger import add_log
+from pymonetdb import connect
+from ..Settings.iotlogger import add_log
 
 Connection = None
 
@@ -10,9 +10,8 @@ def init_monetdb_connection(hostname, po
 global Connection
 
 try:  # the autocommit is set to true so each statement will be independent
-Connection = pymonetdb.connect(hostname=hostname, port=port, 
username=user_name, password=user_password,
-   database=database, autocommit=True)
-Connection.execute("SET SCHEMA iot;")
+Connection = connect(hostname=hostname, port=port, username=user_name, 
password=user_password,
+ database=database, autocommit=True)
 log_message = 'User %s connected successfully to database %s' % 
(user_name, database)
 print log_message
 add_log(20, log_message)
@@ -26,16 +25,26 @@ def close_monetdb_connection():
 Connection.close()
 
 
+def check_hugeint_type():
+Connection.execute("START TRANSACTION")
+cursor = Connection.cursor()
+cursor.execute("SELECT COUNT(*) FROM sys.types WHERE sqlname='hugeint'")
+result = cursor.fetchall()[0][0]
+Connection.commit()
+return result
+
+
 def fetch_streams():
-try:  # TODO paginate results?
+try:
 cursor = Connection.cursor()
 sql_string = """SELECT schemas."name" as schema, tables."name" as 
table, columns."name" as column,
  columns."type", columns."type_digits", columns."type_scale", 
columns."default", columns."null" FROM
  (SELECT "id", "name", "schema_id" FROM sys.tables WHERE type=4) 
AS tables INNER JOIN (SELECT "id", "name"
- FROM sys.schemas) AS schemas ON (tables."schema_id"=schemas."id") 
INNER JOIN  (SELECT "table_id", "name",
+ FROM sys.schemas) AS schemas ON (tables."schema_id"=schemas."id") 
INNER JOIN (SELECT "table_id", "name",
  "type", "type_digits", "type_scale", "default", "null" FROM 
sys.columns) AS columns ON
- (columns."table_id"=tables."id");""".replace('\n', ' ')  # 
important STREAM TABLES TYPE is 4
+ (columns."table_id"=tables."id")""".replace('\n', ' ')
 cursor.execute(sql_string)
 return cursor.fetchall()
 except BaseException as ex:
 add_log(50, ex)
+raise
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,7 +1,7 @@
 import struct
+
 from abc import ABCMeta, abstractmethod
 from datetime import date, time, datetime
-
 from dateutil.relativedelta import relativedelta
 
 LITTLE_ENDIAN_ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
diff --git a/clients/iotapi/src/Streams/streampolling.py 

MonetDB: iot - Fixed imports, changed mapi driver

[Pedro Ferreira]

Changeset: ccf07922520a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ccf07922520a
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Settings/mapiconnection.py
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Fixed imports, changed mapi driver


diffs (105 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -1,10 +1,10 @@
 git+https://github.com/dpallot/simple-websocket-server.git
 IPy==0.83
 jsonschema==2.5.1
-pymonetdb==0.1.1
 python-dateutil==2.5.3
+python-monetdb==11.24.0
 pytz==2016.4
-Sphinx==1.4.3
+Sphinx==1.4.4
 sphinx-rtd-theme==0.1.9
 tzlocal==1.2.2
 watchdog==0.8.3
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -1,6 +1,6 @@
 import sys
 
-from pymonetdb import connect
+from monetdb.sql import connect
 from ..Settings.iotlogger import add_log
 
 Connection = None
diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -3,11 +3,11 @@ Flask-RESTful==0.3.5
 IPy==0.83
 iso8601==0.1.11
 jsonschema==2.5.1
-pymonetdb==0.1.1
 python-dateutil==2.5.3
+python-monetdb==11.24.0
 pytz==2016.4
 rfc3987==1.3.6
-Sphinx==1.4.3
+Sphinx==1.4.4
 sphinx-rtd-theme==0.1.9
 strict-rfc3339==0.7
 tzlocal==1.2.2
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -1,4 +1,4 @@
-from pymonetdb import connect
+from monetdb.sql import connect
 from Settings.iotlogger import add_log
 
 
diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -9,7 +9,7 @@ from itertools import chain
 from json import dumps
 from math import ceil, log10
 from re import compile, split
-from .jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
+from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
 
 # The null constants might change from system to system due to different CPU's 
limits
 LITTLE_ENDIAN_ALIGNMENT = '<'  # for now it is little-endian
diff --git a/clients/iotclient/src/Streams/jsonschemas.py 
b/clients/iotclient/src/Streams/jsonschemas.py
--- a/clients/iotclient/src/Streams/jsonschemas.py
+++ b/clients/iotclient/src/Streams/jsonschemas.py
@@ -1,7 +1,7 @@
 UUID_REGEX = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
 IPV4_REGEX = 
"^(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}(?:\/[0-2]\d|\/3[0-2])?$"
 MAC_ADDRESS_REGEX = "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"
-TIME_REGEX = 
"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\.\d{3}[\+-]([01]\d|2[0-3]):[0-5]\d$"
+TIME_REGEX = 
"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\.\d{3}([\+-]([01]\d|2[0-3]):[0-5]\d)?$"
 
 TIMED_FLUSH_IDENTIFIER = "time"
 TUPLE_FLUSH_IDENTIFIER = "tuple"
diff --git a/clients/iotclient/tests/main.py b/clients/iotclient/tests/main.py
--- a/clients/iotclient/tests/main.py
+++ b/clients/iotclient/tests/main.py
@@ -1,11 +1,11 @@
 import argparse
 import getpass
 import os
-import pymonetdb
 import sys
 import unittest
 
-from .datatypesinsertstests import TextTest, URLTest, INetTest, UUIDTest, 
BooleanTest, TinyIntegerTest,\
+from monetdb.sql import connect
+from datatypesinsertstests import TextTest, URLTest, INetTest, UUIDTest, 
BooleanTest, TinyIntegerTest,\
 SmallIntegerTest, IntegerTest, BigIntegerTest, HugeIntegerTest, 
RealPointTest, FloatPointTest, DecimalTest,\
 DateTest, TimeWithoutTimezoneTest, TimeWithTimezoneTest, 
TimestampWithoutTimezoneTest, TimestampWithTimezoneTest,\
 NullableTextTest, NullableURLTest, NullableINetTest, NullableUUIDTest, 
NullableBooleanTest,\
@@ -54,8 +54,8 @@ def main():
 
 try:
 con_password = getpass.getpass(prompt='Insert password for user ' + 
args['user'] + ':')
-mapi_connection = pymonetdb.connect(hostname=args['host'], 
port=args['port'], username=args['user'],
-password=con_password, 
database=args['database'], autocommit=True)
+mapi_connection = connect(hostname=args['host'], port=args['port'], 
username=args['user'],
+  password=con_password, 
database=args['database'], autocommit=True)
 except BaseException as ex:
 print ex
   

MonetDB: iot - Added python mapi driver support for JSON and UUI...

[Pedro Ferreira]

Changeset: 7d82fa34c0f4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7d82fa34c0f4
Modified Files:
clients/python2/monetdb/sql/pythonize.py
clients/python2/monetdb/sql/types.py
clients/python3/monetdb/sql/pythonize.py
clients/python3/monetdb/sql/types.py
Branch: iot
Log Message:

Added python mapi driver support for JSON and UUID types


diffs (170 lines):

diff --git a/clients/python2/monetdb/sql/pythonize.py 
b/clients/python2/monetdb/sql/pythonize.py
--- a/clients/python2/monetdb/sql/pythonize.py
+++ b/clients/python2/monetdb/sql/pythonize.py
@@ -27,6 +27,7 @@ def _extract_timezone(data):
 
 return data[:-6], datetime.timedelta(hours=sign * int(data[-5:-3]), 
minutes=sign * int(data[-2:]))
 
+
 def strip(data):
 """ returns a python string, with chopped off quotes,
 and replaced escape characters"""
@@ -71,6 +72,7 @@ def py_timestamp(data):
 else:
 return datetime.datetime.strptime(data, '%Y-%m-%d %H:%M:%S')
 
+
 def py_timestamptz(data):
 """ Returns a python Timestamp where data contains a tz code
 """
@@ -86,31 +88,33 @@ mapping = {
 types.VARCHAR: strip,
 types.CLOB: strip,
 types.BLOB: str,
-types.DECIMAL: Decimal,
+types.WRD: int,
+types.SERIAL: int,
+types.TINYINT: int,
 types.SMALLINT: int,
+types.SHORTINT: int,
+types.MEDIUMINT: int,
 types.INT: int,
-types.WRD: int,
+types.LONGINT: int,
 types.BIGINT: int,
 types.HUGEINT: int,
-types.SERIAL: int,
 types.REAL: float,
+types.FLOAT: float,
 types.DOUBLE: float,
+types.DECIMAL: Decimal,
 types.BOOLEAN: py_bool,
 types.DATE: py_date,
 types.TIME: py_time,
+types.TIMETZ: py_timetz,
 types.TIMESTAMP: py_timestamp,
 types.TIMESTAMPTZ: py_timestamptz,
-types.TIMETZ: py_timetz,
 types.INTERVAL: strip,
 types.MONTH_INTERVAL: strip,
 types.SEC_INTERVAL: strip,
-types.TINYINT: int,
-types.SHORTINT: int,
-types.MEDIUMINT: int,
-types.LONGINT: int,
-types.FLOAT: float,
 types.URL: strip,
 types.INET: str,
+types.UUID: strip,
+types.JSON: strip
 }
 
 
diff --git a/clients/python2/monetdb/sql/types.py 
b/clients/python2/monetdb/sql/types.py
--- a/clients/python2/monetdb/sql/types.py
+++ b/clients/python2/monetdb/sql/types.py
@@ -34,6 +34,8 @@ TINYINT = 'tinyint'
 
 URL = 'url'
 INET = 'inet'
+UUID = 'uuid'
+JSON = 'json'
 
 # Not on the website:
 SHORTINT = 'shortint'
@@ -47,7 +49,9 @@ TIMETZ = 'timetz'
 # full names and aliases, spaces are replaced with underscores
 CHARACTER = CHAR
 CHARACTER_VARYING = VARCHAR
-CHARACHTER_LARGE_OBJECT = CLOB
+CHARACTER_LARGE_OBJECT = CLOB
 BINARY_LARGE_OBJECT = BLOB
 NUMERIC = DECIMAL
 DOUBLE_PRECISION = DOUBLE
+BOOL = BOOLEAN
+INTEGER = INT
diff --git a/clients/python3/monetdb/sql/pythonize.py 
b/clients/python3/monetdb/sql/pythonize.py
--- a/clients/python3/monetdb/sql/pythonize.py
+++ b/clients/python3/monetdb/sql/pythonize.py
@@ -28,6 +28,7 @@ def _extract_timezone(data):
 
 return data[:-6], datetime.timedelta(hours=sign * int(data[-5:-3]), 
minutes=sign * int(data[-2:]))
 
+
 def strip(data):
 """ returns a python string, with chopped off quotes,
 and replaced escape characters"""
@@ -75,6 +76,7 @@ def py_timestamp(data):
 else:
 return datetime.datetime.strptime(data, '%Y-%m-%d %H:%M:%S')
 
+
 def py_timestamptz(data):
 """ Returns a python Timestamp where data contains a tz code
 """
@@ -90,31 +92,33 @@ mapping = {
 types.VARCHAR: strip,
 types.CLOB: strip,
 types.BLOB: str,
-types.DECIMAL: Decimal,
+types.WRD: int,
+types.SERIAL: int,
+types.TINYINT: int,
 types.SMALLINT: int,
+types.SHORTINT: int,
+types.MEDIUMINT: int,
 types.INT: int,
-types.WRD: int,
+types.LONGINT: int,
 types.BIGINT: int,
 types.HUGEINT: int,
-types.SERIAL: int,
 types.REAL: float,
+types.FLOAT: float,
 types.DOUBLE: float,
+types.DECIMAL: Decimal,
 types.BOOLEAN: py_bool,
 types.DATE: py_date,
 types.TIME: py_time,
+types.TIMETZ: py_timetz,
 types.TIMESTAMP: py_timestamp,
 types.TIMESTAMPTZ: py_timestamptz,
-types.TIMETZ: py_timetz,
 types.INTERVAL: strip,
 types.MONTH_INTERVAL: strip,
 types.SEC_INTERVAL: strip,
-types.TINYINT: int,
-types.SHORTINT: int,
-types.MEDIUMINT: int,
-types.LONGINT: int,
-types.FLOAT: float,
 types.URL: strip,
 types.INET: str,
+types.UUID: strip,
+types.JSON: strip
 }
 
 
diff --git a/clients/python3/monetdb/sql/types.py 
b/clients/python3/monetdb/sql/types.py
--- a/clients/python3/monetdb/sql/types.py
+++ b/clients/python3/monetdb/sql/types.py
@@ -34,6 +34,8 @@ TINYINT = 'tinyint'
 
 URL = 'url'
 INET = 'inet'
+UUID = 'uuid'
+JSON = 'json'
 
 # Not on the website:
 SHORTINT = 'shortint'
@@ -47,7 +49,9 @@ TIMETZ = 'timetz'
 # full names and aliases, spaces are replaced with underscores
 

MonetDB: iot - Several bugfixes while reading from the database

[Pedro Ferreira]

Changeset: 0dbef0f0ddf6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0dbef0f0ddf6
Modified Files:
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/customthreading.py
sql/backends/monet5/iot/50_iot.sql
Branch: iot
Log Message:

Several bugfixes while reading from the database


diffs (truncated from 1062 to 300 lines):

diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,6 +1,7 @@
 import struct
 
 from abc import ABCMeta, abstractmethod
+from collections import OrderedDict
 from datetime import date, time, datetime
 from dateutil.relativedelta import relativedelta
 
@@ -57,10 +58,8 @@ class StreamDataType(object):
 return results
 
 def to_json_representation(self):  # get a json representation of the data 
type while checking the stream's info
-dic = {'name': self._column_name, 'type': self._data_type, 'nullable': 
self._is_nullable}
-if self._default_value is not None:
-dic['default'] = self._default_value
-return dic
+return OrderedDict((('name', self._column_name), ('type', 
self._data_type),
+('default', self._default_value), ('nullable', 
self._is_nullable)))
 
 
 class TextType(StreamDataType):
@@ -329,7 +328,7 @@ class TimeType(StreamDataType):  # Store
 return results
 
 
-class TimestampType(StreamDataType):  # it's represented with the two integers 
from time and date
+class TimestampType(StreamDataType):  # It is represented with the two 
integers from time and date
 """Covers: TIMESTAMP"""
 
 def __init__(self, *args):
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -65,8 +65,8 @@ class IOTStream:
 return self._stream_name
 
 def get_data_dictionary(self):
-dic = OrderedDict({'schema': self._schema_name, 'stream': 
self._stream_name,
-   'columns': [value.to_json_representation() for 
value in self._columns.values()]})
+dic = OrderedDict((('schema', self._schema_name), ('stream', 
self._stream_name),
+   ('columns', [value.to_json_representation() for 
value in self._columns.values()])))
 self._baskets_lock.acquire_read()
 count = len(self._baskets)
 listing = [{'number': k, 'count': v} for k, v in self._baskets.items()]
@@ -146,4 +146,5 @@ class IOTStream:
 
 keys = results.keys()  # TODO check if this is viable for many tuples!!
 tuples = [dict(zip(keys, values)) for values in zip(*(results[k] for k 
in keys))]
-return {'schema': self._schema_name, 'stream': self._stream_name, 
'count': read_tuples, 'tuples': tuples}
+return OrderedDict((('schema', self._schema_name), ('stream', 
self._stream_name), ('count', read_tuples),
+('tuples', tuples)))
diff --git a/clients/iotapi/src/Streams/streamscontext.py 
b/clients/iotapi/src/Streams/streamscontext.py
--- a/clients/iotapi/src/Streams/streamscontext.py
+++ b/clients/iotapi/src/Streams/streamscontext.py
@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from Utilities.readwritelock import RWLock
 from WebSockets.websockets import unsubscribe_removed_streams
 
@@ -39,8 +40,8 @@ class IOTStreams:
 
 def get_streams_data(self):
 self._locker.acquire_read()
-res = {'streams_count': len(self._context),
-   'streams_listing': [value.get_data_dictionary() for value in 
self._context.values()]}
+res = OrderedDict((('streams_count', len(self._context)),
+   ('streams_listing', [value.get_data_dictionary() 
for value in self._context.values()])))
 self._locker.release()
 return res
 
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -2,7 +2,7 @@ from datetime import datetime
 from flask import request
 from flask_restful import Resource
 from json import loads
-from jsonschema import Draft4Validator, FormatChecker
+from jsonschema import Draft4Validator, FormatChecker, ValidationError
 from pytz 

MonetDB: iot - Added interval type support, updated iotapi polling

[Pedro Ferreira]

Changeset: 9b8b8136fbc1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9b8b8136fbc1
Modified Files:
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotapi/src/main.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Added interval type support, updated iotapi polling


diffs (truncated from 1848 to 300 lines):

diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -6,7 +6,7 @@ from .iotlogger import add_log
 Baskets_Location = None
 
 if sys.platform in ("linux", "linux2", "darwin"):
-DEFAULT_FILESYSTEM = '/etc/iotapi'
+DEFAULT_FILESYSTEM = '/var/iotapi'
 elif sys.platform == "win32":
 DEFAULT_FILESYSTEM = os.path.join(os.path.dirname(__file__), os.pardir)
 
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -1,50 +1,44 @@
-import sys
-
 from monetdb.sql import connect
-from ..Settings.iotlogger import add_log
-
-Connection = None
+from .iotlogger import add_log
 
 
 def init_monetdb_connection(hostname, port, user_name, user_password, 
database):
-global Connection
+return connect(hostname=hostname, port=port, username=user_name, 
password=user_password, database=database)
 
-try:  # the autocommit is set to true so each statement will be independent
-Connection = connect(hostname=hostname, port=port, username=user_name, 
password=user_password,
- database=database, autocommit=True)
-log_message = 'User %s connected successfully to database %s' % 
(user_name, database)
-print log_message
-add_log(20, log_message)
-except BaseException as ex:
-print ex
-add_log(50, ex)
-sys.exit(1)
 
+def close_monetdb_connection(connection):
+connection.close()
 
-def close_monetdb_connection():
-Connection.close()
 
-
-def check_hugeint_type():
-Connection.execute("START TRANSACTION")
-cursor = Connection.cursor()
+def check_hugeint_type(connection):
+cursor = connection.cursor()
 cursor.execute("SELECT COUNT(*) FROM sys.types WHERE sqlname='hugeint'")
 result = cursor.fetchall()[0][0]
-Connection.commit()
-return result
+connection.commit()
+return result > 0
 
 
-def fetch_streams():
+def mapi_get_database_streams(connection):
 try:
-cursor = Connection.cursor()
-sql_string = """SELECT schemas."name" as schema, tables."name" as 
table, columns."name" as column,
- columns."type", columns."type_digits", columns."type_scale", 
columns."default", columns."null" FROM
+cursor = connection.cursor()
+sql_string = """SELECT tables."id", schemas."name" AS schema, 
tables."name" AS table FROM
  (SELECT "id", "name", "schema_id" FROM sys.tables WHERE type=4) 
AS tables INNER JOIN (SELECT "id", "name"
- FROM sys.schemas) AS schemas ON (tables."schema_id"=schemas."id") 
INNER JOIN (SELECT "table_id", "name",
- "type", "type_digits", "type_scale", "default", "null" FROM 
sys.columns) AS columns ON
- (columns."table_id"=tables."id")""".replace('\n', ' ')
+ FROM sys.schemas) AS schemas ON 
(tables."schema_id"=schemas."id")""".replace('\n', ' ')
 cursor.execute(sql_string)
-return cursor.fetchall()
+tables = cursor.fetchall()
+
+cursor = connection.cursor()
+sql_string = """SELECT columns."table_id", columns."name" AS column, 
columns."type", columns."type_digits",
+columns."type_scale", columns."default", columns."null" FROM 
(SELECT "table_id", "name", "type",
+"type_digits", "type_scale", "default", "null", "number" FROM 
sys.columns) AS columns INNER JOIN
+(SELECT "id" FROM sys.tables WHERE type=4) AS tables ON 
(tables."id"=columns."table_id")
+ORDER BY columns."table_id", columns."number" """.replace('\n', ' 
')
+cursor.execute(sql_string)
+columns = cursor.fetchall()
+
+

MonetDB: iot - Merged with backend code

[Pedro Ferreira]

Changeset: 30283d4f77ec for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=30283d4f77ec
Removed Files:
sql/backends/monet5/iot/Tests/iot01.sql
Modified Files:
monetdb5/mal/mal_instruction.c
monetdb5/optimizer/opt_iot.c
sql/backends/monet5/iot/50_iot.sql
sql/backends/monet5/iot/Makefile.am
sql/backends/monet5/iot/Tests/All
sql/backends/monet5/iot/Tests/cleanup.sql
sql/backends/monet5/iot/Tests/iot00.sql
sql/backends/monet5/iot/Tests/iot02.sql
sql/backends/monet5/iot/Tests/iot03.sql
sql/backends/monet5/iot/Tests/iot10.sql
sql/backends/monet5/iot/Tests/iot12.sql
sql/backends/monet5/iot/Tests/iot15.sql
sql/backends/monet5/iot/Tests/receptor01.sql
sql/backends/monet5/iot/basket.c
sql/backends/monet5/iot/basket.h
sql/backends/monet5/iot/basket.mal
sql/backends/monet5/iot/iot.c
sql/backends/monet5/iot/iot.h
sql/backends/monet5/iot/iot.mal
sql/backends/monet5/iot/petrinet.c
sql/backends/monet5/iot/petrinet.h
sql/backends/monet5/iot/petrinet.mal
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_scenario.c
Branch: iot
Log Message:

Merged with backend code


diffs (truncated from 1908 to 300 lines):

diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -207,6 +207,7 @@ freeMalBlk(MalBlkPtr mb)
 
if (mb->history)
freeMalBlk(mb->history);
+   mb->history = 0;
if (mb->binding)
GDKfree(mb->binding);
mb->binding = 0;
diff --git a/monetdb5/optimizer/opt_iot.c b/monetdb5/optimizer/opt_iot.c
--- a/monetdb5/optimizer/opt_iot.c
+++ b/monetdb5/optimizer/opt_iot.c
@@ -34,7 +34,7 @@
 #include "opt_dataflow.h"
 
 #define MAXBSKT 64
-#define isstream(S,T) \
+#define getStreamTableInfo(S,T) \
for(fnd=0, k= 0; k< btop; k++) \
if( strcmp(schemas[k], S)== 0 && strcmp(tables[k], T)== 0 ){ \
fnd= 1; break;\
@@ -50,13 +50,18 @@ OPTiotImplementation(Client cntxt, MalBl
str  tables[MAXBSKT];
int  mvc[MAXBSKT];
int done[MAXBSKT]= {0};
-   int btop=0;
+   int btop=0, lastmvc=0;
int noerror=0;
-   int cq;
+   int cq= strncmp(getFunctionId(getInstrPtr(mb,0)),"cq",2) == 0;
+   char buf[256];
+   lng usec = GDKusec();
 
(void) pci;
 
-   cq= strncmp(getFunctionId(getInstrPtr(mb,0)),"cq",2) == 0;
+   OPTDEBUGiot {
+   mnstr_printf(cntxt->fdout, "#iot optimizer start\n");
+   printFunction(cntxt->fdout, mb, stk, LIST_MAL_DEBUG);
+   } 
old = mb->stmt;
limit = mb->stop;
slimit = mb->ssize;
@@ -64,29 +69,27 @@ OPTiotImplementation(Client cntxt, MalBl
/* first analyse the query for streaming tables */
for (i = 1; i < limit && btop fdout, "#iot stream 
table %s.%s\n", getModuleId(p), getFunctionId(p));
-   schemas[btop]= getVarConstant(mb, getArg(p,1)).val.sval;
-   tables[btop]= getVarConstant(mb, getArg(p,2)).val.sval;
-   mvc[btop] = getArg(p,0);
+   schemas[btop]= getVarConstant(mb, getArg(p,2)).val.sval;
+   tables[btop]= getVarConstant(mb, getArg(p,3)).val.sval;
for( j =0; j< btop ; j++)
if( strcmp(schemas[j], schemas[j+1])==0  && 
strcmp(tables[j],tables[j+1]) ==0)
break;
-   mvc[j] = getArg(p,0);
+   lastmvc = mvc[j] = getArg(p,0);
done[j]= done[j] || getFunctionId(p)== registerRef;
if( j == btop)
btop++;
}
-   if( getModuleId(p)== basketRef && (getFunctionId(p) == 
appendRef || getFunctionId(p) == deleteRef )){
+   if( getModuleId(p)== basketRef && getFunctionId(p) == appendRef 
){
OPTDEBUGiot mnstr_printf(cntxt->fdout, "#iot stream 
table %s.%s\n", getModuleId(p), getFunctionId(p));
schemas[btop]= getVarConstant(mb, getArg(p,2)).val.sval;
tables[btop]= getVarConstant(mb, getArg(p,3)).val.sval;
-   mvc[btop] = getArg(p,0);
for( j =0; j< btop ; j++)
if( strcmp(schemas[j], schemas[j+1])==0  && 
strcmp(tables[j],tables[j+1]) ==0)
  

MonetDB: iot - Fixed imports, tests and time types

[Pedro Ferreira]

Changeset: c069c8238a35 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c069c8238a35
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Fixed imports, tests and time types


diffs (truncated from 592 to 300 lines):

diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,7 +1,6 @@
 fake-factory==0.5.7
 Flask-RESTful==0.3.5
 IPy==0.83
-iso8601==0.1.11
 jsonschema==2.5.1
 python-dateutil==2.5.3
 python-monetdb==11.24.0
diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -5,12 +5,12 @@ from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime, timedelta
 from dateutil import parser
-from iso8601 import parse_date
 from itertools import chain
 from json import dumps
 from math import ceil, log10
 from re import compile, split
-from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
+from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX, 
TIME_WITH_TIMEZONE_TYPE_INTERNAL,\
+TIME_WITH_TIMEZONE_TYPE_EXTERNAL, TIMESTAMP_WITH_TIMEZONE_TYPE_INTERNAL, 
TIMESTAMP_WITH_TIMEZONE_TYPE_EXTERNAL
 
 # The null constants might change from system to system due to different CPU's 
limits
 LITTLE_ENDIAN_ALIGNMENT = '<'  # for now it is little-endian
@@ -671,18 +671,24 @@ class DateType(BaseDateTimeType):  # Sto
 return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter) + 'I', 
*extracted_values)
 
 
-class TimeWithoutTimeZoneType(BaseDateTimeType):  # Stored as an uint with the 
number of milliseconds since hour00:00:00
+class TimeType(BaseDateTimeType):  # Stored as an uint with the number of 
milliseconds since hour00:00:00
 """Covers: TIME"""
 
 def __init__(self, **kwargs):
-super(TimeWithoutTimeZoneType, self).__init__(**kwargs)
+super(TimeType, self).__init__(**kwargs)
 
 def add_json_schema_entry(self, schema):
-super(TimeWithoutTimeZoneType, self).add_json_schema_entry(schema)
+super(TimeType, self).add_json_schema_entry(schema)
 schema[self._column_name]['pattern'] = TIME_REGEX
 
 def parse_entry(self, entry):
-return parser.parse(entry)
+parsed = parser.parse(entry)
+string = parsed.strftime("%z")
+delta = timedelta(hours=int(string[1:3]), minutes=int(string[3:5]))
+if string[0] == '-':
+delta = -delta
+parsed = parsed.replace(tzinfo=None) - delta
+return parsed
 
 def pack_next_value(self, parsed, counter, parameters, errors):
 if parsed is None:
@@ -694,12 +700,22 @@ class TimeWithoutTimeZoneType(BaseDateTi
 def pack_parsed_values(self, extracted_values, counter, parameters):
 return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter) + 'I', 
*extracted_values)
 
+def to_json_representation(self):
+json_value = super(TimeType, self).to_json_representation()
+if self._data_type == TIME_WITH_TIMEZONE_TYPE_INTERNAL:
+json_value['type'] = TIME_WITH_TIMEZONE_TYPE_EXTERNAL
+return json_value
 
-class TimeWithTimeZoneType(TimeWithoutTimeZoneType):
-"""Covers: TIME WITH TIME ZONE"""
+
+class TimestampType(BaseDateTimeType):  # It is represented with the two 
integers from time and date
+"""Covers: TIMESTAMP"""
 
 def __init__(self, **kwargs):
-super(TimeWithTimeZoneType, self).__init__(**kwargs)
+super(TimestampType, self).__init__(**kwargs)
+
+def add_json_schema_entry(self, schema):
+super(TimestampType, self).add_json_schema_entry(schema)
+schema[self._column_name]['format'] = 'date-time'
 
 def parse_entry(self, entry):
 parsed = parser.parse(entry)
@@ -710,25 +726,6 @@ class TimeWithTimeZoneType(TimeWithoutTi
 parsed = parsed.replace(tzinfo=None) - delta
 return parsed
 
-def to_json_representation(self):
-json_value = super(TimeWithTimeZoneType, self).to_json_representation()
-json_value['type'] = 'time with time zone'
-return json_value
-
-
-class TimestampWithoutTimeZoneType(BaseDateTimeType):  # It is represented 
with the two integers from time and date
-"""Covers: TIMESTAMP"""
-
-def __init__(self, **kwargs):
-super(TimestampWithoutTimeZoneType, self).__init__(**kwargs)
-
-def add_json_schema_entry(self, schema):
-super(TimestampWithoutTimeZoneType, self).add_json_schema_entry(schema)
-schema[self._column_name]['format'] = 

MonetDB: iot - Changed polling SQL query. Added more information...

[Pedro Ferreira]

Changeset: 18a241a95c44 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=18a241a95c44
Modified Files:
clients/iotapi/documentation/websockets_api.rst
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotclient/src/Streams/datatypes.py
Branch: iot
Log Message:

Changed polling SQL query. Added more information about streams' columns on Web 
API


diffs (truncated from 358 to 300 lines):

diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -185,11 +185,14 @@ Message with details about a stream incl
 "columns": [
 {
 "name": "sensorid",
-"type": "clob"
+"type": "clob",
+"nullable": false,
+"default": "living room"
 },
 {
 "name": "temperature",
-"type": "real"
+"type": "real",
+"nullable": false
 }
 ],
 "baskets_count": 3,
@@ -225,11 +228,14 @@ Returns a info message regarding all the
 "columns": [
 {
 "name": "sensorid",
-"type": "clob"
+"type": "clob",
+"nullable": false,
+"default": "living room"
 },
 {
 "name": "temperature",
-"type": "real"
+"type": "real",
+"nullable": false
 }
 ],
 "baskets_count": 3,
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -1,4 +1,3 @@
-import getpass
 import sys
 
 import pymonetdb
@@ -30,11 +29,12 @@ def close_monetdb_connection():
 def fetch_streams():
 try:  # TODO paginate results?
 cursor = Connection.cursor()
-sql_string = """SELECT storage."schema", storage."table", 
storage."column", storage."type", storage."typewidth"
-  FROM (SELECT "schema", "table", "column", "type", "typewidth" FROM 
sys.storage) AS storage
-  INNER JOIN (SELECT "name" FROM sys.tables WHERE type=4) AS tables ON 
(storage."table"=tables."name")
-  INNER JOIN (SELECT "name" FROM sys.schemas) AS schemas ON 
(storage."schema"=schemas."name");"""\
-.replace('\n', ' ')
+sql_string = """SELECT schemas."name" as schema, tables."name" as 
table, columns."name" as column,
+ columns."type", columns."type_digits", columns."type_scale", 
columns."default", columns."null" FROM
+ (SELECT "id", "name", "schema_id" FROM sys.tables WHERE type=4) 
AS tables INNER JOIN (SELECT "id", "name"
+ FROM sys.schemas) AS schemas ON (tables."schema_id"=schemas."id") 
INNER JOIN  (SELECT "table_id", "name",
+ "type", "type_digits", "type_scale", "default", "null" FROM 
sys.columns) AS columns ON
+ (columns."table_id"=tables."id");""".replace('\n', ' ')  # 
important STREAM TABLES TYPE is 4
 cursor.execute(sql_string)
 return cursor.fetchall()
 except BaseException as ex:
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -20,13 +20,17 @@ FLOAT_NAN = struct.unpack('f', '\xff\xff
 DOUBLE_NAN = struct.unpack('d', '\xff\xff\xff\xff\xff\xff\xef\xff')[0]
 
 
+# elem[0] is column name, elem[1] is type, elem[2] is type_digits, elem[3] is 
type_scale elem[4] is default value
+# elem[5] is nullable
 class StreamDataType(object):
 """MonetDB's data types for reading base class"""
 __metaclass__ = ABCMeta
 
-def __init__(self, **kwargs):
-self._column_name = kwargs['name']  # name of the column
-self._data_type = kwargs['type']  # SQL name of the type
+def __init__(self, *args):
+self._column_name = args[0]  # name of the column
+self._data_type = args[1]  # SQL name of the type
+self._default_value = args[4]  # default value text
+self._is_nullable = args[5]  # is nullable
 
 def is_file_mode_binary(self):
 return True
@@ -53,14 +57,17 @@ class StreamDataType(object):
 return results
 
 def to_json_representation(self):  # get a json representation of the data 
type while checking the stream's info
-return {'name': self._column_name, 'type': self._data_type}
+dic = {'name': self._column_name, 'type': self._data_type, 'nullable': 
self._is_nullable}
+if self._default_value is not None:

MonetDB: iot - Improved arguments parsing

[Pedro Ferreira]

Changeset: 811ec4cbbf29 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=811ec4cbbf29
Modified Files:
clients/iotapi/documentation/api_server_arguments.rst
clients/iotapi/requirements.txt
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Improved arguments parsing


diffs (truncated from 681 to 300 lines):

diff --git a/clients/iotapi/documentation/api_server_arguments.rst 
b/clients/iotapi/documentation/api_server_arguments.rst
--- a/clients/iotapi/documentation/api_server_arguments.rst
+++ b/clients/iotapi/documentation/api_server_arguments.rst
@@ -24,31 +24,30 @@ Paths
 
 .. important:: Make sure the current user has permissions on the locations 
provided.
 
-**-f - -filesystem=**
+**-f - -filesystem**
 
 Set the filesystem directory where the baskets will be created. By default in 
UNIX systems is on :code:`/etc/iotapi` directory, while on Windows is on the 
directory where the :code:`main.py` script was invoked.
 
-**-l  - -log=**
+**-l  - -log**
 
 Location of logfile. On the logfile is reported when streams are created or 
removed, when tuples are inserted and when the baskets are flushed. By default 
in UNIX systems is :code:`/var/log/iot/iotapi.log`, while on Windows is the 
:code:`iotapi.log` on the directory where the :code:`main.py` script was called.
 
-
 Web API Listening and Behavior
 --
 
 Customize the Web API server parameters and behavior.
 
-**-sh  - -shost=**
+**-sh  - -shost**
 
 Listening host of the application (IOT) server. By default is on 
:code:`0.0.0.0`.
 
-**-sp  - -sport=**
+**-sp  - -sport**
 
 Listening port of the application (IOT) server. By default is on port 
:code:`8002`.
 
-**-pi  - -polling=**
+**-pi  - -polling**
 
-Set the polling interval to MonetDB database for updates. By default is 
:code:`60` seconds.
+Set the polling interval in seconds to MonetDB database for streams updates. 
By default is :code:`60` seconds.
 
 Database Connection
 ---
@@ -57,18 +56,25 @@ Credentials for the MAPI connection to M
 
 .. note:: The user's password will be prompted during the initialization of 
the server.
 
-**-d  - -host=**
+**-d  - -host**
 
 Listening host of MonetDB database. By default is on :code:`127.0.0.1`.
 
-**-d  - -port=**
+**-d  - -port**
 
 Listening port of MonetDB database. By default is on port :code:`5`.
 
-**-u  - -user=**
+**-u  - -user**
 
 Name of the user to authenticate. By default is user :code:`monetdb`.
 
-**-d  - -database=**
+**-d  - -database**
 
 Name of database to use. By default is :code:`iotdb` database.
+
+Help
+
+
+**- -help**
+
+Display arguments help.
diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -1,4 +1,5 @@
 git+https://github.com/dpallot/simple-websocket-server.git
+IPy==0.83
 jsonschema==2.5.1
 pymonetdb==0.1.1
 python-dateutil==2.5.3
diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -1,22 +1,19 @@
+import os
 import sys
-import os
 
 from iotlogger import add_log
 
 Baskets_Location = None
 
+if sys.platform in ("linux", "linux2", "darwin"):
+DEFAULT_FILESYSTEM = '/etc/iotapi'
+elif sys.platform == "win32":
+DEFAULT_FILESYSTEM = os.path.join(os.path.dirname(__file__), os.pardir)
+
 
 def init_file_system(new_location=None):
 global Baskets_Location
 
-if new_location is None:
-if sys.platform in ("linux", "linux2", "darwin"):
-new_location = '/etc/iotapi'
-elif sys.platform == "win32":
-new_location = os.path.join(os.path.dirname(__file__), os.pardir)
-else:
-new_location = new_location
-
 try:
 Baskets_Location = os.path.join(new_location, "baskets")
 if not os.path.exists(Baskets_Location):
diff --git a/clients/iotapi/src/Settings/iotlogger.py 
b/clients/iotapi/src/Settings/iotlogger.py
--- a/clients/iotapi/src/Settings/iotlogger.py
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -1,35 +1,27 @@
 import logging
+import os
 import sys
 
-import os
-
 Logger = logging.getLogger("IOTAPILog")
 
+if sys.platform in ("linux", "linux2", "darwin"):
+DEFAULT_LOGGING = '/var/log/iot/iotapi.log'
+elif sys.platform == "win32":
+DEFAULT_LOGGING = os.path.join(os.path.dirname(__file__), os.pardir, 
'iotapi.log')
 
-def init_logging(new_location):
-

MonetDB: iot - Small corrections on arguments listening

[Pedro Ferreira]

Changeset: 1f3d04c409fe for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1f3d04c409fe
Modified Files:
clients/iotapi/documentation/api_server_arguments.rst
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/src/main.py
Branch: iot
Log Message:

Small corrections on arguments listening


diffs (104 lines):

diff --git a/clients/iotapi/documentation/api_server_arguments.rst 
b/clients/iotapi/documentation/api_server_arguments.rst
--- a/clients/iotapi/documentation/api_server_arguments.rst
+++ b/clients/iotapi/documentation/api_server_arguments.rst
@@ -45,7 +45,7 @@ Listening host of the application (IOT) 
 
 Listening port of the application (IOT) server. By default is on port 
:code:`8002`.
 
-**-pi  - -polling**
+**-po  - -polling**
 
 Set the polling interval in seconds to MonetDB database for streams updates. 
By default is :code:`60` seconds.
 
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -62,19 +62,19 @@ def main():
 parser = argparse.ArgumentParser(description='IOT Web API for MonetDB', 
epilog="There might exist bugs!",
  add_help=False)
 parser.add_argument('-f', '--filesystem', type=check_path, nargs='?', 
default=DEFAULT_FILESYSTEM,
-help='Baskets\' location directory (default: %s)' % 
DEFAULT_FILESYSTEM)
+help='Baskets location directory (default: %s)' % 
DEFAULT_FILESYSTEM, metavar='DIRECTORY')
 parser.add_argument('-l', '--log', type=check_path, nargs='?', 
default=DEFAULT_LOGGING,
-help='Logging file location (default: %s)' % 
DEFAULT_LOGGING)
-parser.add_argument('-pi', '--polling', type=check_positive_int, 
nargs='?', default=60,
+help='Logging file location (default: %s)' % 
DEFAULT_LOGGING, metavar='FILE_PATH')
+parser.add_argument('-po', '--polling', type=check_positive_int, 
nargs='?', default=60,
 help='Polling interval in seconds to the database for 
streams updates (default: 60)')
 parser.add_argument('-sh', '--shost', type=check_ipv4_address, nargs='?', 
default='0.0.0.0',
-help='Web API server host (default: 0.0.0.0)')
+help='Web API server host (default: 0.0.0.0)', 
metavar='HOST')
 parser.add_argument('-sp', '--sport', type=check_positive_int, nargs='?', 
default=8002,
-help='Web API server port (default: 8002)')
+help='Web API server port (default: 8002)', 
metavar='PORT')
 parser.add_argument('-h', '--host', nargs='?', default='127.0.0.1',
-help='MonetDB database host (default: 127.0.0.1)')
+help='MonetDB database host (default: 127.0.0.1)', 
metavar='HOST')
 parser.add_argument('-p', '--port', type=check_positive_int, nargs='?', 
default=5,
-help='Database listening port (default: 5)')
+help='Database listening port (default: 5)', 
metavar='PORT')
 parser.add_argument('-d', '--database', nargs='?', default='iotdb', 
help='Database name (default: iotdb)')
 parser.add_argument('-u', '--user', nargs='?', default='monetdb', 
help='Database user (default: monetdb)')
 parser.add_argument('--help', action='store_true', help='Display this 
help')
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -35,13 +35,13 @@ Location of logfile. On the logfile is r
 Host Identifier
 ---
 
-If the *useidentifier* parameter is provided, an extra column on streams will 
be added with a custom name of the host for later identification. 
+If the *identifier* parameter is provided, an extra column on streams will be 
added with a custom name of the host for later identification.
 
-**-ui  - -useidentifier**
+**-i  - -identifier**
 
 Use a host identifier for every new stream.
 
-**-in  - -name**
+**-n  - -name**
 
 Host identifier name. By default is the host's MAC address.
 
diff --git a/clients/iotclient/src/main.py b/clients/iotclient/src/main.py
--- a/clients/iotclient/src/main.py
+++ b/clients/iotclient/src/main.py
@@ -72,26 +72,26 @@ def main():
 parser = argparse.ArgumentParser(description='IOT Web Server for MonetDB', 
epilog="There might exist bugs!",
  add_help=False)
 parser.add_argument('-f', '--filesystem', type=check_path, nargs='?', 
default=DEFAULT_FILESYSTEM,
-help='Baskets\' location directory (default: %s)' % 
DEFAULT_FILESYSTEM)
+help='Baskets location directory (default: %s)' % 

MonetDB: iot - Starting MonetDB integration

[Pedro Ferreira]

Changeset: a4516136d717 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a4516136d717
Added Files:
clients/iotclient/src/Streams/streampolling.py
Modified Files:
clients/iotapi/documentation/api_server_arguments.rst
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/main.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
sql/backends/monet5/iot/50_iot.sql
sql/backends/monet5/iot/basket.c
sql/backends/monet5/sql.c
Branch: iot
Log Message:

Starting MonetDB integration


diffs (truncated from 523 to 300 lines):

diff --git a/clients/iotapi/documentation/api_server_arguments.rst 
b/clients/iotapi/documentation/api_server_arguments.rst
--- a/clients/iotapi/documentation/api_server_arguments.rst
+++ b/clients/iotapi/documentation/api_server_arguments.rst
@@ -75,6 +75,6 @@ Name of database to use. By default is :
 Help
 
 
-**-he  - -help**
+**-?  - -help**
 
-Display arguments help.
+Display help.
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -77,7 +77,7 @@ def main():
 help='Database listening port (default: 5)', 
metavar='PORT')
 parser.add_argument('-d', '--database', nargs='?', default='iotdb', 
help='Database name (default: iotdb)')
 parser.add_argument('-u', '--user', nargs='?', default='monetdb', 
help='Database user (default: monetdb)')
-parser.add_argument('-he', '--help', action='store_true', help='Display 
this help')
+parser.add_argument('-?', '--help', action='store_true', help='Display 
this help')
 
 try:
 args = vars(parser.parse_args())
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -92,6 +92,6 @@ Name of database to use. By default is :
 Help
 
 
-**-he  - -help**
+**-?  - -help**
 
-Display arguments help.
+Display help.
diff --git a/clients/iotclient/src/Settings/filesystem.py 
b/clients/iotclient/src/Settings/filesystem.py
--- a/clients/iotclient/src/Settings/filesystem.py
+++ b/clients/iotclient/src/Settings/filesystem.py
@@ -5,7 +5,6 @@ from iotlogger import add_log
 from Utilities.filecreator import create_file_if_not_exists
 
 Baskets_Location = None
-Configfile_Location = None
 
 if sys.platform in ("linux", "linux2", "darwin"):
 DEFAULT_FILESYSTEM = '/etc/iotserver'
@@ -14,14 +13,12 @@ elif sys.platform == "win32":
 
 
 def init_file_system(filesystem_location):
-global Baskets_Location, Configfile_Location
+global Baskets_Location
 
 try:
 Baskets_Location = os.path.join(filesystem_location, "baskets")
 if not os.path.exists(Baskets_Location):
 os.makedirs(Baskets_Location)
-Configfile_Location = 
create_file_if_not_exists(os.path.join(filesystem_location, "config.json"),
-init_text='[]')
 except (Exception, OSError) as ex:
 print ex
 add_log(50, ex)
@@ -30,7 +27,3 @@ def init_file_system(filesystem_location
 
 def get_baskets_location():
 return Baskets_Location
-
-
-def get_configfile_location():
-return Configfile_Location
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -11,9 +11,9 @@ def init_monetdb_connection(hostname, po
 
 try:  # the autocommit is set to true so each statement will be independent
 Connection = pymonetdb.connect(hostname=hostname, port=port, 
username=user_name, password=user_password,
-   database=database, autocommit=True)
+   database=database, autocommit=False)
 log_message = 'User %s connected successfully to database %s' % 
(user_name, database)
-print >> sys.stdout, log_message
+print log_message
 add_log(20, log_message)
 except BaseException as ex:
 print ex
@@ -24,52 +24,67 @@ def init_monetdb_connection(hostname, po
 def close_monetdb_connection():
 Connection.close()
 
-
-def mapi_fetch_all_streams():
+
+def mapi_get_webserver_streams():
 try:
+Connection.execute("BEGIN TRANSACTION")
 cursor = Connection.cursor()
-sql = ''.join(['SELECT storage."schema", storage."table", 
storage."column", storage."type", storage',
-   '."typewidth" FROM (SELECT 

MonetDB: iot - Finished first version of documentation for web a...

[Pedro Ferreira]

Changeset: 0c61a0ebc2b9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c61a0ebc2b9
Modified Files:
clients/iotapi/documentation/websockets_api.rst
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/WebSockets/websockets.py
Branch: iot
Log Message:

Finished first version of documentation for web api. Added pattern for response 
messages during a WebSocket session.


diffs (truncated from 401 to 300 lines):

diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -18,7 +18,7 @@ The following sections explain the avail
 sub
 ---
 
-Subscribes for new basket creations from a specific stream. Whenever a basket 
is created, the server sends a notification message indicating the number of 
inserted tuples in the new basket. The user has to specify the stream's name 
and schema.
+Subscribes for new basket creations from a specific stream. Whenever a basket 
is created, the server sends a notification message indicating the number of 
inserted tuples in the new basket. The user has to specify the stream's name 
and schema. To subscribe to a temperature stream, the following would suffice:
 
 .. code-block:: json
 
@@ -33,6 +33,14 @@ unsub
 
 Unsubscribes a previous subscribed stream for a client. The user has to 
specify the stream's name and schema. The example is the same as above, just 
changing the request keyword.
 
+.. code-block:: json
+
+{
+"request": "unsub",
+"schema": "measures",
+"stream": "temperature"
+}
+
 read
 
 
@@ -40,7 +48,7 @@ Reads output result from baskets generat
 
 .. important:: The user has not to be subscribed to the stream in order to 
read data from it!
 
-It's possible to provide an offset, a limit and a basket number where the read 
should start. The request will always provide a result, even if the query 
provides no tuples to read. The user has to specify the stream's name and 
schema.
+It's possible to provide an offset, a limit and a basket number where the read 
should start. The request will always provide a result, even if the query 
provides no tuples to read. The user has to specify the stream's name and 
schema. A possible query for the temperatures stream:
 
 .. code-block:: json
 
@@ -54,7 +62,7 @@ It's possible to provide an offset, a li
 info
 
 
-Retrieves information about a giving stream if a stream's name and schema are 
provided, or all the existing streams in the system otherwise.
+Retrieves information about a giving stream if a stream's name and schema are 
provided, or all the existing streams in the system otherwise. To request 
information of temperature stream:
 
 .. code-block:: json
 
@@ -66,3 +74,178 @@ Retrieves information about a giving str
 
 Responses
 =
+
+The response messages, are also provided with JSON. They contain a 
:code:`response` field, identifying the type of the message.
+
+error
+-
+
+An internal error occurred in the server during a client's request. The 
:code:`message` contains a string explaining the error. The following error 
message happens when the user attempts to unsubscribed to a non subscribed 
stream:
+
+.. code-block:: json
+
+{
+"response": "error",
+"message": "Stream measures.temperature not present in the user's 
subscriptions!"
+}
+
+subscribed
+--
+
+Message confirming the subscription to new baskets notifications of a stream.
+
+.. code-block:: json
+
+{
+"response": "subscribed",
+"schema": "measures",
+"stream": "temperature"
+}
+
+unsubscribed
+
+
+Message confirming the removal of a subscription to new baskets notifications 
of a stream.
+
+.. code-block:: json
+
+{
+"response": "unsubscribed",
+"schema": "measures",
+"stream": "temperature"
+}
+
+removed
+---
+
+If a stream is removed on MonetDB's engine, but the client is still 
subscribed, then this message is sent.
+
+.. code-block:: json
+
+{
+"response": "removed",
+"schema": "measures",
+"stream": "temperature"
+}
+
+notification
+
+
+Notification of a new basket creation from a subscribed stream. The message 
contains the basket number and the number of tuples in the new basket. A 
possible notification example for the above stream:
+
+.. code-block:: json
+
+{
+"response": "notification",
+"schema": "measures",
+"stream": "temperature",
+"basket": 2,
+"count": 50
+}
+
+read
+
+
+Response message for a read query. Contains the reconstructed tuples 
listening. The tuples are listened in the say they are inserted in the RESTful 
webserver. If a column has a null value, the JSON's :code:`null` value will be 
listened. A possible 

MonetDB: iot - Small corrections

[Pedro Ferreira]

Changeset: bd0abb9d0ed4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bd0abb9d0ed4
Modified Files:
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/streampolling.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Small corrections


diffs (161 lines):

diff --git a/clients/iotapi/src/Settings/iotlogger.py 
b/clients/iotapi/src/Settings/iotlogger.py
--- a/clients/iotapi/src/Settings/iotlogger.py
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -3,6 +3,7 @@ import os
 import sys
 
 Logger = logging.getLogger("IOTAPILog")
+Logger.setLevel(logging.INFO)
 
 if sys.platform in ("linux", "linux2", "darwin"):
 DEFAULT_LOGGING = '/var/log/iot/iotapi.log'
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -14,10 +14,10 @@ def init_monetdb_connection(hostname, po
database=database, autocommit=True)
 Connection.execute("SET SCHEMA iot;")
 log_message = 'User %s connected successfully to database %s' % 
(user_name, database)
-print >> sys.stdout, log_message
+print log_message
 add_log(20, log_message)
 except BaseException as ex:
-print >> sys.stdout, ex
+print ex
 add_log(50, ex)
 sys.exit(1)
 
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -33,20 +33,21 @@ def stream_polling():
 new_streams = {}
 array = fetch_streams()  # TODO check whenever stream's columns are updated
 
-for key, group in groupby(array, lambda x: 
Streams_Context.get_context_entry_name(x[0], x[1])):
-if key not in current_streams:
-columns = {}
+if array is not None:
+for key, group in groupby(array, lambda x: 
Streams_Context.get_context_entry_name(x[0], x[1])):
+if key not in current_streams:
+columns = {}
 
-for elem in group:
-for entry in SWITCHER:  # allocate the proper type wrapper
-if elem[3] in entry['types']:
-reflection_class = globals()[entry['class']]  # import 
everything from datatypes!!!
-new_column = reflection_class(*elem[2:])
-columns[elem[2]] = new_column  # add new column to the 
dictionary
-break
-new_streams[key] = IOTStream(schema_name=elem[0], 
stream_name=elem[1], columns=columns)
-else:
-retained_streams.append(key)
+for elem in group:
+for entry in SWITCHER:  # allocate the proper type wrapper
+if elem[3] in entry['types']:
+reflection_class = globals()[entry['class']]  # 
import everything from datatypes!!!
+new_column = reflection_class(*elem[2:])
+columns[elem[2]] = new_column  # add new column to 
the dictionary
+break
+new_streams[key] = IOTStream(schema_name=elem[0], 
stream_name=elem[1], columns=columns)
+else:
+retained_streams.append(key)
 
 retained_streams_final = [key for key in current_streams if key in 
retained_streams]
 Streams_Context.merge_context(retained_streams_final, new_streams)
diff --git a/clients/iotclient/src/Settings/iotlogger.py 
b/clients/iotclient/src/Settings/iotlogger.py
--- a/clients/iotclient/src/Settings/iotlogger.py
+++ b/clients/iotclient/src/Settings/iotlogger.py
@@ -3,6 +3,7 @@ import sys
 import os
 
 Logger = logging.getLogger("IOTServerLog")
+Logger.setLevel(logging.INFO)
 
 if sys.platform in ("linux", "linux2", "darwin"):
 DEFAULT_LOGGING = '/var/log/iot/iotserver.log'
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -60,12 +60,8 @@ def mapi_create_stream(schema, stream, c
 Connection.execute("CREATE SCHEMA " + schema + ";")
 except:
 pass
-
-try:  # attempt to create the stream table
-Connection.execute("SET SCHEMA " + schema + ";")
-Connection.execute(''.join(["CREATE STREAM TABLE ", stream, " (", 
columns, ");"]))
-except BaseException as ex:
-add_log(40, ex)
+Connection.execute("SET SCHEMA " + schema + ";")
+Connection.execute(''.join(["CREATE STREAM 

MonetDB: iot - Added nullable constants verification tests

[Pedro Ferreira]

Changeset: 2708b772e8db for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2708b772e8db
Added Files:
clients/iotclient/tests/datatypesinsertstests.py
Removed Files:
clients/iotclient/tests/datatypestests.py
Modified Files:
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Added nullable constants verification tests


diffs (truncated from 511 to 300 lines):

diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -16,12 +16,12 @@ LITTLE_ENDIAN_ALIGNMENT = '<'  # for now
 NIL_STRING = "\200"
 NIL_UUID = "----"
 
-INT8_MIN = 0x80
-INT16_MIN = 0x8000
-INT32_MIN = 0x8000
-INT64_MIN = 0x8000
-INT64_MAX = 0x
-INT128_MIN = 0x8000
+INT8_MIN = -128
+INT16_MIN = -32768
+INT32_MIN = -2147483648
+INT64_MIN = -9223372036854775808
+INT64_MAX = +9223372036854775807
+INT128_MIN = -340282366920938463463374607431768211456
 
 FLOAT_NAN = struct.unpack('f', '\xff\xff\x7f\xff')[0]
 DOUBLE_NAN = struct.unpack('d', '\xff\xff\xff\xff\xff\xff\xef\xff')[0]
@@ -370,6 +370,8 @@ class BooleanType(StreamDataType):
 return INT8_MIN
 
 def process_next_value(self, entry, counter, parameters, errors):
+if entry == self.get_nullable_constant():
+return self.get_nullable_constant()
 if bool(entry):
 return 1
 return 0
@@ -464,10 +466,10 @@ class HugeIntegerType(NumberBaseType):
 schema[self._column_name]['type'] = 'integer'
 
 def get_nullable_constant(self):
-return INT128_MIN
+return 0x8000
 
 def process_next_value(self, entry, counter, parameters, errors):
-return [entry & INT64_MAX, (entry >> 64) & INT64_MAX]
+return [entry & 0x, (entry >> 64) & 0x]
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
 extracted_values = list(itertools.chain(*extracted_values))
@@ -536,9 +538,10 @@ class DecimalType(NumberBaseType):
 .get(self._pack_sym)
 
 def check_value_precision(self, value, text):
-number_digits = int(math.ceil(math.log10(abs(value
-if number_digits > self._precision:
-raise Exception('Too many digits on %s: %s > %s!' % (text, 
number_digits, self._precision))
+if value != self._nullable_constant:
+number_digits = int(math.ceil(math.log10(abs(value
+if number_digits > self._precision:
+raise Exception('Too many digits on %s: %s > %s!' % (text, 
number_digits, self._precision))
 
 def add_json_schema_entry(self, schema):
 super(DecimalType, self).add_json_schema_entry(schema)
@@ -552,7 +555,7 @@ class DecimalType(NumberBaseType):
 if self._pack_sym != 'Q':
 return int(entry)
 else:
-return [int(entry) & INT64_MAX, (int(entry) >> 64) & INT64_MAX]
+return [int(entry) & 0x, (int(entry) >> 64) & 
0x]
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
 if self._pack_sym == 'Q':
@@ -647,7 +650,7 @@ class DateType(BaseDateTimeType):  # Sto
 
 def pack_next_value(self, parsed, counter, parameters, errors):
 if parsed is None:
-return INT32_MIN
+return 0x8000
 day0 = copy.deepcopy(parsed).replace(year=1, month=1, day=1)
 # the minyear in python is 1, but for the representation is 0, so why 
the add
 return int((parsed - day0).days) + 366
@@ -682,7 +685,7 @@ class TimeType(BaseDateTimeType):  # Sto
 
 def pack_next_value(self, parsed, counter, parameters, errors):
 if parsed is None:
-return INT32_MIN
+return 0x8000
 hour0 = copy.deepcopy(parsed).replace(hour=0, minute=0, second=0, 
microsecond=0)
 delta = parsed - hour0
 return int(delta.total_seconds()) * 1000 + int(delta.microseconds) / 
1000
@@ -726,7 +729,7 @@ class TimestampType(BaseDateTimeType):  
 
 def pack_next_value(self, parsed, counter, parameters, errors):
 if parsed is None:
-return [0, INT32_MIN]
+return [0, 0x8000]
 hour0 = copy.deepcopy(parsed).replace(hour=0, minute=0, second=0, 
microsecond=0)
 day0 = copy.deepcopy(parsed).replace(year=1, month=1, day=1)
 days = int((parsed - day0).days) + 366
diff --git a/clients/iotclient/tests/datatypestests.py 
b/clients/iotclient/tests/datatypesinsertstests.py
rename from clients/iotclient/tests/datatypestests.py
rename to clients/iotclient/tests/datatypesinsertstests.py
--- a/clients/iotclient/tests/datatypestests.py
+++ b/clients/iotclient/tests/datatypesinsertstests.py
@@ -170,6 +170,7 

MonetDB: iot - Added first batch of tests

[Pedro Ferreira]

Changeset: 891059aea396 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=891059aea396
Added Files:
clients/iotclient/tests/__init__.py
clients/iotclient/tests/datatypestests.py
clients/iotclient/tests/main.py
Modified Files:
clients/iotapi/documentation/api_server_arguments.rst
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Added first batch of tests


diffs (truncated from 774 to 300 lines):

diff --git a/clients/iotapi/documentation/api_server_arguments.rst 
b/clients/iotapi/documentation/api_server_arguments.rst
--- a/clients/iotapi/documentation/api_server_arguments.rst
+++ b/clients/iotapi/documentation/api_server_arguments.rst
@@ -75,6 +75,6 @@ Name of database to use. By default is :
 Help
 
 
-**- -help**
+**-he  - -help**
 
 Display arguments help.
diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -11,11 +11,11 @@ elif sys.platform == "win32":
 DEFAULT_FILESYSTEM = os.path.join(os.path.dirname(__file__), os.pardir)
 
 
-def init_file_system(new_location=None):
+def init_file_system(filesystem_location):
 global Baskets_Location
 
 try:
-Baskets_Location = os.path.join(new_location, "baskets")
+Baskets_Location = os.path.join(filesystem_location, "baskets")
 if not os.path.exists(Baskets_Location):
 os.makedirs(Baskets_Location)
 except (Exception, OSError) as ex:
diff --git a/clients/iotapi/src/Settings/iotlogger.py 
b/clients/iotapi/src/Settings/iotlogger.py
--- a/clients/iotapi/src/Settings/iotlogger.py
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -12,13 +12,11 @@ elif sys.platform == "win32":
 
 def init_logging(logging_location):
 try:
-Logger.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s 
- %(message)s')
 logging_path = os.path.dirname(logging_location)
 if not os.path.exists(logging_path):
 os.makedirs(logging_path)
 log_handler = logging.FileHandler(logging_location, mode='a+')
-log_handler.setFormatter(formatter)
+log_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - 
%(levelname)s - %(message)s'))
 Logger.addHandler(log_handler)
 except (Exception, OSError) as ex:
 print ex
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -77,7 +77,7 @@ def main():
 help='Database listening port (default: 5)', 
metavar='PORT')
 parser.add_argument('-d', '--database', nargs='?', default='iotdb', 
help='Database name (default: iotdb)')
 parser.add_argument('-u', '--user', nargs='?', default='monetdb', 
help='Database user (default: monetdb)')
-parser.add_argument('--help', action='store_true', help='Display this 
help')
+parser.add_argument('-he', '--help', action='store_true', help='Display 
this help')
 
 try:
 args = vars(parser.parse_args())
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -92,6 +92,6 @@ Name of database to use. By default is :
 Help
 
 
-**- -help**
+**-he  - -help**
 
 Display arguments help.
diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,8 +1,10 @@
+python-dateutil==2.5.3
+fake-factory==0.5.7
 Flask-RESTful==0.3.5
 IPy==0.83
+iso8601==0.1.11
 jsonschema==2.5.1
 pymonetdb==0.1.1
-python-dateutil==2.5.3
 pytz==2016.4
 rfc3987==1.3.6
 Sphinx==1.4.1
diff --git a/clients/iotclient/src/Settings/iotlogger.py 
b/clients/iotclient/src/Settings/iotlogger.py
--- a/clients/iotclient/src/Settings/iotlogger.py
+++ b/clients/iotclient/src/Settings/iotlogger.py
@@ -12,13 +12,11 @@ elif sys.platform == "win32":
 
 def init_logging(logging_location):
 try:
-Logger.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s 
- %(message)s')
 logging_path = os.path.dirname(logging_location)
 if not os.path.exists(logging_path):
 os.makedirs(logging_path)
 log_handler = 

MonetDB: iot - Merged commits

[Pedro Ferreira]

Changeset: ff4753ccd11c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ff4753ccd11c
Added Files:
MacOSX/MonetDB_logo.png
clients/R/Tests/dplyr.timeout
clients/R/Tests/install.timeout
debian/fix-deb.sh
debian/monetdb-r.install
gdk/interprocess.c
gdk/interprocess.h
java/tests/Test_CisValid.java
java/tests/Test_FetchSize.java
monetdb5/extras/pyapi/50_pyapi.mal
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/Tests/All
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.malC
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.stable.err
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.stable.out
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.malC
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.err
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
monetdb5/extras/pyapi/Tests/pyapi_pandas.malC
monetdb5/extras/pyapi/Tests/pyapi_pandas.stable.err
monetdb5/extras/pyapi/Tests/pyapi_pandas.stable.out
monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_huge.malC
monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.malC
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_string.malC
monetdb5/extras/pyapi/Tests/pyapi_types_string.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_string.stable.out
monetdb5/extras/pyapi/connection.c
monetdb5/extras/pyapi/connection.h
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/formatinput.h
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
monetdb5/extras/pyapi/pyapi_locatepython.bat
monetdb5/extras/pyapi/pytypes.c
monetdb5/extras/pyapi/pytypes.h
monetdb5/extras/pyapi/type_conversion.c
monetdb5/extras/pyapi/type_conversion.h
monetdb5/extras/pyapi/unicode.c
monetdb5/extras/pyapi/unicode.h
monetdb5/extras/pyapi/unspecified_evil.h
monetdb5/modules/mal/Tests/orderidx04.stable.err
monetdb5/modules/mal/Tests/orderidx04.stable.out
sql/backends/monet5/Tests/pyapi00.sql
sql/backends/monet5/Tests/pyapi00.stable.err
sql/backends/monet5/Tests/pyapi00.stable.out
sql/backends/monet5/Tests/pyapi01.sql
sql/backends/monet5/Tests/pyapi01.stable.err
sql/backends/monet5/Tests/pyapi01.stable.out
sql/backends/monet5/Tests/pyapi02.sql
sql/backends/monet5/Tests/pyapi02.stable.err
sql/backends/monet5/Tests/pyapi02.stable.out
sql/backends/monet5/Tests/pyapi03.sql
sql/backends/monet5/Tests/pyapi03.stable.err
sql/backends/monet5/Tests/pyapi03.stable.out
sql/backends/monet5/Tests/pyapi04.sql
sql/backends/monet5/Tests/pyapi04.stable.err
sql/backends/monet5/Tests/pyapi04.stable.out
sql/backends/monet5/Tests/pyapi05.sql
sql/backends/monet5/Tests/pyapi05.stable.err
sql/backends/monet5/Tests/pyapi05.stable.out
sql/backends/monet5/Tests/pyapi06.sql
sql/backends/monet5/Tests/pyapi06.stable.err
sql/backends/monet5/Tests/pyapi06.stable.out
sql/backends/monet5/Tests/pyapi07.sql
sql/backends/monet5/Tests/pyapi07.stable.err
sql/backends/monet5/Tests/pyapi07.stable.out
sql/backends/monet5/Tests/pyapi08.sql
sql/backends/monet5/Tests/pyapi08.stable.err
sql/backends/monet5/Tests/pyapi08.stable.out
sql/backends/monet5/Tests/pyapi09.sql
sql/backends/monet5/Tests/pyapi09.stable.err
sql/backends/monet5/Tests/pyapi09.stable.out
sql/backends/monet5/Tests/pyapi10.sql
sql/backends/monet5/Tests/pyapi10.stable.err
sql/backends/monet5/Tests/pyapi10.stable.err.Windows
sql/backends/monet5/Tests/pyapi10.stable.out
sql/backends/monet5/Tests/pyapi11.sql
sql/backends/monet5/Tests/pyapi11.stable.err
sql/backends/monet5/Tests/pyapi11.stable.out
sql/backends/monet5/Tests/pyapi12.sql
sql/backends/monet5/Tests/pyapi12.stable.err
sql/backends/monet5/Tests/pyapi12.stable.out
sql/backends/monet5/Tests/pyapi13.sql
sql/backends/monet5/Tests/pyapi13.stable.err

MonetDB: iot - Fixed front end test

[Pedro Ferreira]

Changeset: 635c867ab029 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=635c867ab029
Added Files:
clients/iotapi/tests/websocketclient.py
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/WebSockets/jsonschemas.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotapi/tests/frontendtests.py
clients/iotapi/tests/main.py
clients/iotclient/requirements.txt
Branch: iot
Log Message:

Fixed front end test


diffs (truncated from 449 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -1,12 +1,12 @@
 git+https://github.com/dpallot/simple-websocket-server.git
-IPy==0.83
-jsonschema==2.5.1
-python-dateutil==2.5.3
-python-monetdb==11.24.0
-pytz==2016.4
-requests==2.10.0
-Sphinx==1.4.4
-sphinx-rtd-theme==0.1.9
-tzlocal==1.2.2
-watchdog==0.8.3
-websocket-client==0.37.0
+IPy>=0.83
+jsonschema>=2.5.1
+python-dateutil>=2.5.3
+python-monetdb>=11.19.3.2
+pytz>=2016.4
+requests>=2.10.0
+Sphinx>=1.4.4
+sphinx-rtd-theme>=0.1.9
+tornado>=3.2.2
+tzlocal>=1.2.2
+watchdog>=0.8.3
diff --git a/clients/iotapi/src/WebSockets/jsonschemas.py 
b/clients/iotapi/src/WebSockets/jsonschemas.py
--- a/clients/iotapi/src/WebSockets/jsonschemas.py
+++ b/clients/iotapi/src/WebSockets/jsonschemas.py
@@ -1,37 +1,35 @@
 SUBSCRIBE_OPTS = ["sub", "subscribe"]
 UNSUBSCRIBE_OPTS = ["unsub", "unsubscribe"]
 INFO_OPTS = ["info"]
-CONCAT_SUB_OPTS = SUBSCRIBE_OPTS + UNSUBSCRIBE_OPTS
 READ_OPTS = ["read"]
 
 CLIENTS_INPUTS_SCHEMA = {
-"title": "JSON schema publish/subscribe streams",
+"title": "JSON schema fro web api",
 "description": "Validate clients inputs",
 "$schema": "http://json-schema.org/draft-04/schema#;,
 "type": "object",
-
 "anyOf": [{
 "properties": {
+"request": {"type": "string", "enum": SUBSCRIBE_OPTS + 
UNSUBSCRIBE_OPTS + INFO_OPTS},
 "schema": {"type": "string"},
-"stream": {"type": "string"},
-"request": {"type": "string", "enum": CONCAT_SUB_OPTS + INFO_OPTS},
+"stream": {"type": "string"}
 },
-"required": ["schema", "stream", "request"],
+"required": ["request", "schema", "stream"],
 "additionalProperties": False
 }, {
 "properties": {
+"request": {"type": "string", "enum": READ_OPTS},
 "schema": {"type": "string"},
 "stream": {"type": "string"},
-"request": {"type": "string", "enum": READ_OPTS},
-"basket": {"type": "integer", "minimum": 1, "default": 1},
+"basket": {"type": "integer", "minimum": 1},
 "limit": {"type": "integer", "minimum": 0, "default": 100},
 "offset": {"type": "integer", "minimum": 0, "default": 0}
 },
-"required": ["schema", "stream", "request"],
+"required": ["request", "schema", "stream", "basket"],
 "additionalProperties": False
 }, {
 "properties": {
-"request": {"type": "string", "enum": INFO_OPTS}  # for all streams
+"request": {"type": "string", "enum": INFO_OPTS}
 },
 "required": ["request"],
 "additionalProperties": False
diff --git a/clients/iotapi/src/WebSockets/websockets.py 
b/clients/iotapi/src/WebSockets/websockets.py
--- a/clients/iotapi/src/WebSockets/websockets.py
+++ b/clients/iotapi/src/WebSockets/websockets.py
@@ -68,10 +68,10 @@ class IOTAPI(WebSocket):
 elif input_schema['request'] in UNSUBSCRIBE_OPTS:
 self.unsubscribe(input_schema['schema'], 
input_schema['stream'])
 elif input_schema['request'] in READ_OPTS:
-basket = input_schema.get('basket', 1)
 limit = input_schema.get('limit', 100)
 offset = input_schema.get('offset', 0)
-self.read_stream_batch(input_schema['schema'], 
input_schema['stream'], basket, limit, offset)
+self.read_stream_batch(input_schema['schema'], 
input_schema['stream'],
+   input_schema['basket'], limit, offset)
 elif input_schema['request'] in INFO_OPTS:
 if len(input_schema) == 1:  # get all streams information
 self.get_streams_data()
@@ -88,7 +88,7 @@ class IOTAPI(WebSocket):
 self._subscriptions[concatenated_name] = stream
 self._subscriptions_locker.release()
 self.sendJSONMessage((('response', 'subscribed'), ('schema', 
schema_name), ('stream', stream_name)))
-add_log(20, ''.join(['Client ', self.address[0], 'subscribed to stream 
', concatenated_name]))
+add_log(20, ''.join(['Client ', self.address[0], ' subscribed to 
stream ', concatenated_name]))
 
 def unsubscribe(self, schema_name, stream_name):
 concatenated_name = IOTStreams.get_context_entry_name(schema_name, 

MonetDB: iot - Updated documentation, fixed some output listings

[Pedro Ferreira]

Changeset: 12a303d540c3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=12a303d540c3
Added Files:
clients/iotapi/documentation/about.rst
clients/iotclient/documentation/about.rst
Removed Files:
clients/iotapi/documentation/conclusion.rst
clients/iotclient/documentation/conclusion.rst
Modified Files:
clients/iotapi/documentation/index.rst
clients/iotapi/documentation/introduction.rst
clients/iotapi/documentation/websockets_api.rst
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotclient/documentation/index.rst
clients/iotclient/documentation/introduction.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
Branch: iot
Log Message:

Updated documentation, fixed some output listings


diffs (truncated from 629 to 300 lines):

diff --git a/clients/iotapi/documentation/conclusion.rst 
b/clients/iotapi/documentation/about.rst
rename from clients/iotapi/documentation/conclusion.rst
rename to clients/iotapi/documentation/about.rst
--- a/clients/iotapi/documentation/conclusion.rst
+++ b/clients/iotapi/documentation/about.rst
@@ -1,18 +1,8 @@
-.. _future_work:
-
-**
-Final Acknowledgements
-**
-
-Future Work
-===
-
-Currently the project is on a testing phase.
-
 .. _about-the-project:
 
+*
 About the Project
-=
+*
 
 .. _Send questions and suggestions to e-mail.: p.e.ferre...@cwi.nl
 
diff --git a/clients/iotapi/documentation/index.rst 
b/clients/iotapi/documentation/index.rst
--- a/clients/iotapi/documentation/index.rst
+++ b/clients/iotapi/documentation/index.rst
@@ -16,4 +16,4 @@ Contents:
introduction.rst
api_server_arguments.rst
websockets_api.rst
-   conclusion.rst
+   about.rst
diff --git a/clients/iotapi/documentation/introduction.rst 
b/clients/iotapi/documentation/introduction.rst
--- a/clients/iotapi/documentation/introduction.rst
+++ b/clients/iotapi/documentation/introduction.rst
@@ -1,4 +1,4 @@
-.. _instroduction:
+.. _introduction:
 
 
 Introduction
diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -158,15 +158,18 @@ Response message for a read query. Conta
 "tuples": [
 {
 "sensorid": "living room",
-"temperature": 32.6
+"temperature": 32.6,
+"implicit_timestamp": "2016-06-17T09:23:22+00:00"
 },
 {
 "sensorid": "kitchen",
-"temperature": 34.2
+"temperature": 34.2,
+"implicit_timestamp": "2016-06-17T09:23:22+00:00"
 },
 {
 "sensorid": "bathroom",
-"temperature": 28.9
+"temperature": 28.9,
+"implicit_timestamp": "2016-06-17T09:23:22+00:00"
 }
 ]
 }
@@ -192,7 +195,14 @@ Message with details about a stream incl
 {
 "name": "temperature",
 "type": "real",
-"nullable": false
+"nullable": false,
+"default": null
+},
+{
+"name": "implicit_timestamp",
+"type": "timestamp with time zone",
+"nullable": false,
+"default": null
 }
 ],
 "baskets_count": 3,
@@ -235,7 +245,14 @@ Returns a info message regarding all the
 {
 "name": "temperature",
 "type": "real",
-"nullable": false
+"nullable": false,
+"default": null
+},
+{
+"name": "implicit_timestamp",
+"type": "timestamp with time zone",
+"nullable": false,
+"default": null
 }
 ],
 "baskets_count": 3,
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -27,6 +27,7 @@ Switcher = [{'types': ['clob', 'url'], '
 Types_Preferences = [['boolean', 'tinyint'], ['smallint'], ['int', 'real', 
'date', 'time', 'timetz', 'month_interval'],
   

MonetDB: iot - Added first Front-End test

[Pedro Ferreira]

Changeset: dce31e77 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dce31e77
Added Files:
clients/iotapi/tests/__init__.py
clients/iotapi/tests/frontendtests.py
clients/iotapi/tests/main.py
Modified Files:
clients/iotapi/requirements.txt
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/tests/main.py
Branch: iot
Log Message:

Added first Front-End test


diffs (truncated from 364 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -4,7 +4,9 @@ jsonschema==2.5.1
 python-dateutil==2.5.3
 python-monetdb==11.24.0
 pytz==2016.4
+requests==2.10.0
 Sphinx==1.4.4
 sphinx-rtd-theme==0.1.9
 tzlocal==1.2.2
 watchdog==0.8.3
+websocket-client==0.37.0
diff --git a/clients/iotapi/tests/__init__.py b/clients/iotapi/tests/__init__.py
new file mode 100644
diff --git a/clients/iotapi/tests/frontendtests.py 
b/clients/iotapi/tests/frontendtests.py
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/tests/frontendtests.py
@@ -0,0 +1,148 @@
+import json
+import os
+import requests
+import websocket
+
+from distutils.dir_util import copy_tree
+from threading import Thread
+from time import sleep
+from unittest import TestCase
+
+__all__ = ['NullablesTest']
+
+WEB_SOCKETS_THREADS_TIMEOUT = 15
+
+
+class BaseFrontEndTest(TestCase):
+
+def __init__(self, **kwargs):
+super(BaseFrontEndTest, self).__init__()
+self._web_server_baskets_location = 
os.path.join(kwargs['iot_client_path'], 'baskets')
+self._web_api_baskets_location = os.path.join(kwargs['iot_api_path'], 
'baskets')
+self.schema = "tests"
+
+def export_inserts(self, schema, stream, basket):
+input_dir = os.path.join(self._web_server_baskets_location, schema, 
stream, basket)
+output_dir = os.path.join(self._web_api_baskets_location, schema, 
stream, basket)
+copy_tree(input_dir, output_dir)
+
+
+class TestWebSocket(websocket.WebSocketApp):
+
+def __init__(self, test, url, header=[], on_open=None, on_message=None, 
on_error=None, on_close=None, on_ping=None,
+ on_pong=None, on_cont_message=None, keep_running=True, 
get_mask_key=None, cookie=None,
+ subprotocols=None, on_data=None):
+super(TestWebSocket, self).__init__(url, header, on_open, on_message, 
on_error, on_close, on_ping,
+on_pong, on_cont_message, 
keep_running, get_mask_key, cookie,
+subprotocols, on_data)
+self.test = test
+self.ws_state = 1
+
+
+def on_open(ws):
+ws.send(json.dumps({"request": "sub", "schema": ws.test.schema, "stream":  
ws.test.stream}))
+
+
+def on_message(ws, message):
+resp = json.loads(message)
+if resp['response'] == 'error':
+ws.test.fail(msg=resp['message'])
+elif ws.ws_state == 1:
+ws.test.assertDictEqual({"response": "subscribed", "schema": 
ws.test.schema, "stream": ws.test.stream}, resp)
+ws.ws_state = 2
+elif ws.ws_state == 2:
+ws.test.assertDictEqual({"response": "notification", "schema": 
ws.test.schema, "stream": ws.test.stream,
+ "basket": 1, "count": 3}, resp)
+ws.send(json.dumps({"request": "read", "schema": ws.test.schema, 
"stream": ws.test.stream,
+"basket": 1, "offset": 0, "limit": 3}))
+ws.ws_state = 3
+elif ws.ws_state == 3:
+ws.test.assertIn('implicit_timestamp', resp['tuples'][0], 
msg='Timestamp not in stream')
+del resp['tuples'][0]['implicit_timestamp']
+del resp['tuples'][1]['implicit_timestamp']
+del resp['tuples'][2]['implicit_timestamp']
+res_dic = {'vala': None, "valb": None, "valc": None, "vald": None, 
"vale": None, "valf": None, "valg": None,
+   "valh": None, "vali": None, "valj": None, "valk": None, 
"vall": None, "valm": None, "valn": None,
+   "valo": None, "valp": None, "valq": None, "valr": None, 
"vals": None, "valt": None, "valu": None,
+   "valv": None, "valw": None, "valx": None, "valy": None, 
"valz": None}
+tuples_response = {"response": "read", "schema": ws.test.schema, 
"stream": ws.test.stream,
+   "count": 3, "tuples": [res_dic, res_dic, res_dic]}
+ws.test.assertDictEqual(tuples_response, resp)
+ws.close()
+elif ws.ws_state == 4:
+ws.test.assertDictEqual({"response": "removed", "schema": 
ws.test.schema, "stream": ws.test.stream}, resp)
+ws.close()
+
+
+def on_error(ws, error):
+ws.close()
+ws.test.fail(msg=error)
+
+
+def web_socket(test):
+ws = TestWebSocket(test=test, url="ws://127.0.0.1:8002/", 
on_message=on_message, on_open=on_open, on_error=on_error)
+test.ws = ws
+  

MonetDB: iot - Merged with backend

[Pedro Ferreira]

Changeset: ac718d7895af for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ac718d7895af
Added Files:
sql/backends/monet5/Tests/pyapi30.sql
sql/backends/monet5/Tests/pyapi30.stable.err
sql/backends/monet5/Tests/pyapi30.stable.out
Removed Files:
clients/python2/ChangeLog
clients/python2/ChangeLog-Archive
clients/python2/ChangeLog.Jun2016
clients/python2/MANIFEST.in
clients/python2/Makefile.ag
clients/python2/README.rst
clients/python2/monetdb/__init__.py
clients/python2/monetdb/control.py
clients/python2/monetdb/exceptions.py
clients/python2/monetdb/mapi.py
clients/python2/monetdb/sql/__init__.py
clients/python2/monetdb/sql/connections.py
clients/python2/monetdb/sql/converters.py
clients/python2/monetdb/sql/cursors.py
clients/python2/monetdb/sql/monetize.py
clients/python2/monetdb/sql/pythonize.py
clients/python2/monetdb/sql/types.py
clients/python2/setup.py
clients/python2/test/capabilities.py
clients/python2/test/dbapi20.py
clients/python2/test/run.cmd
clients/python2/test/run.sh
clients/python2/test/runtests.py
clients/python2/test/test_control.py
clients/python2/test/test_monetize.py
clients/python2/test/test_pythonize.py
clients/python3/ChangeLog
clients/python3/ChangeLog-Archive
clients/python3/ChangeLog.Jun2016
clients/python3/MANIFEST.in
clients/python3/Makefile.ag
clients/python3/README.rst
clients/python3/monetdb/__init__.py
clients/python3/monetdb/control.py
clients/python3/monetdb/exceptions.py
clients/python3/monetdb/mapi.py
clients/python3/monetdb/sql/__init__.py
clients/python3/monetdb/sql/connections.py
clients/python3/monetdb/sql/converters.py
clients/python3/monetdb/sql/cursors.py
clients/python3/monetdb/sql/monetize.py
clients/python3/monetdb/sql/pythonize.py
clients/python3/monetdb/sql/types.py
clients/python3/setup.py
clients/python3/test/capabilities.py
clients/python3/test/dbapi20.py
clients/python3/test/run.cmd
clients/python3/test/run.sh
clients/python3/test/runtests.py
clients/python3/test/test_control.py
clients/python3/test/test_monetize.py
clients/python3/test/test_pythonize.py
monetdb5/tests/BugReports/Tests/no.146.stable.out
sql/test/mapi/Tests/python2_test_monetdb_sql.SQL.bat
sql/test/mapi/Tests/python2_test_monetdb_sql.SQL.sh
sql/test/mapi/Tests/python2_test_monetdb_sql.stable.err
sql/test/mapi/Tests/python2_test_monetdb_sql.stable.out
sql/test/mapi/Tests/python3_test_monetdb_sql.SQL.bat
sql/test/mapi/Tests/python3_test_monetdb_sql.SQL.sh
sql/test/mapi/Tests/python3_test_monetdb_sql.stable.err
sql/test/mapi/Tests/python3_test_monetdb_sql.stable.out
Modified Files:
.hgtags
MonetDB.spec
NT/Makefile
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/autogen/autogen/am.py
buildtools/autogen/autogen/msc.py
clients/Makefile.ag
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/examples/python/basics.py
clients/examples/python/mclient-python2.py
clients/examples/python/mclient-python3.py
clients/examples/python/perf.py
clients/examples/python/sqlsample.py
clients/mapiclient/msqldump.c
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
common/stream/stream.c
configure.ag
debian/changelog
debian/control
debian/fix-deb.sh
gdk/libbat.rc
geom/lib/libgeom.c
geom/monetdb5/geom.c
geom/sql/Makefile.ag
java/Makefile.ag
java/build.properties
java/pom.xml
java/release.txt
java/src/main/java/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
libversions
monetdb5/ChangeLog-Archive
monetdb5/extras/pyapi/pyapi.c
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/url.c
monetdb5/modules/mal/groupby.c
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/tablet.c

MonetDB: iot - Added automatic tuple count for iotapi so there i...

[Pedro Ferreira]

Changeset: 5614092fb16b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5614092fb16b
Modified Files:
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/WebSockets/jsonschemas.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Added automatic tuple count for iotapi so there is no need for an auxiliary 
file now. Fixed some binary conversions while reading baskets in iotapi. Also 
fixed tuple order in JSON responses.


diffs (truncated from 897 to 300 lines):

diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,22 +1,19 @@
 import struct
 
 from abc import ABCMeta, abstractmethod
-from collections import OrderedDict
 from datetime import date, time, datetime
 from dateutil.relativedelta import relativedelta
+from os import SEEK_END
 
 LITTLE_ENDIAN_ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
 
 NIL_STRING = "\200\n"  # added newline for performance
 NIL_UUID = "----"
 
-INT8_MIN = 0x80
-INT16_MIN = 0x8000
-INT32_MIN = 0x8000
-INT64_MIN = 0x8000
-INT64_MAX = 0x
-INT128_MIN = 0x8000
-
+INT8_MIN = -128
+INT16_MIN = -32768
+INT32_MIN = -2147483648
+INT64_MIN = -9223372036854775808
 FLOAT_NAN = struct.unpack('f', '\xff\xff\x7f\xff')[0]
 DOUBLE_NAN = struct.unpack('d', '\xff\xff\xff\xff\xff\xff\xef\xff')[0]
 
@@ -31,10 +28,17 @@ class StreamDataType(object):
 self._default_value = kwargs['default']  # default value text
 self._is_nullable = kwargs['nullable']  # is nullable
 
+def get_column_name(self):
+return self._column_name
+
 def is_file_mode_binary(self):
 return True
 
 @abstractmethod
+def calculate_file_size(self, fp):
+pass
+
+@abstractmethod
 def skip_tuples(self, fp, offset):
 pass
 
@@ -56,12 +60,12 @@ class StreamDataType(object):
 return results
 
 def to_json_representation(self):  # get a json representation of the data 
type while checking the stream's info
-return OrderedDict((('name', self._column_name), ('type', 
self._data_type),
-('default', self._default_value), ('nullable', 
self._is_nullable)))
+return (('name', self._column_name), ('type', self._data_type),
+('default', self._default_value), ('nullable', 
self._is_nullable))
 
 
 class TextType(StreamDataType):
-"""Covers: CLOB and Url"""
+"""Covers: CLOB and URL"""
 
 def __init__(self, **kwargs):
 super(TextType, self).__init__(**kwargs)
@@ -70,6 +74,9 @@ class TextType(StreamDataType):
 def is_file_mode_binary(self):
 return False
 
+def calculate_file_size(self, fp):
+return sum(1 for _ in fp)
+
 def skip_tuples(self, fp, offset):
 for _ in xrange(offset):
 next(fp)
@@ -93,9 +100,7 @@ class LimitedTextType(TextType):
 self._limit = kwargs['digits']
 
 def to_json_representation(self):
-json_value = super(LimitedTextType, self).to_json_representation()
-json_value['limit'] = self._limit
-return json_value
+return super(LimitedTextType, self).to_json_representation() + 
(('limit', self._limit),)
 
 
 class INetType(StreamDataType):
@@ -104,6 +109,10 @@ class INetType(StreamDataType):
 def __init__(self, **kwargs):
 super(INetType, self).__init__(**kwargs)
 
+def calculate_file_size(self, fp):
+fp.seek(0, SEEK_END)
+return fp.tell() >> 3  # a inet has a size of 8 bytes
+
 def skip_tuples(self, fp, offset):
 fp.seek(offset << 3)
 
@@ -129,6 +138,10 @@ class UUIDType(StreamDataType):
 def __init__(self, **kwargs):
 super(UUIDType, self).__init__(**kwargs)
 
+def calculate_file_size(self, fp):
+fp.seek(0, SEEK_END)
+return fp.tell() >> 4  # a inet has a size of 16 bytes
+
 def skip_tuples(self, fp, offset):
 fp.seek(offset << 4)
 
@@ -145,7 +158,6 @@ class UUIDType(StreamDataType):
 next_uuid.append("-")
 else:
 next_uuid.append("%02x" % next(iterator))
-
 built_uuid = ''.join(next_uuid)
 if built_uuid == NIL_UUID:
 built_uuid = None
@@ -158,14 +170,17 @@ class BooleanType(StreamDataType):
 
 def __init__(self, **kwargs):
 super(BooleanType, 

MonetDB: iot - Fixed array description

[Pedro Ferreira]

Changeset: 344882141988 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=344882141988
Modified Files:
clients/iotapi/src/Streams/streampolling.py
Branch: iot
Log Message:

Fixed array description


diffs (35 lines):

diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -21,15 +21,17 @@ Switcher = [{'types': ['clob', 'url'], '
 {'types': ['inet'], 'class': INetType},
 {'types': ['uuid'], 'class': UUIDType}]
 
-# this queue will be use to determine the best type to calculate the output 
size of the baskets
-Queue = [['boolean', 'tinyint'], ['smallint'], ['int', 'real', 'date', 'time', 
'timetz', 'month_interval'],
- ['bigint', 'sec_interval', 'double', 'timestamp', 'timestamptz', 
'inet'], ['decimal'], ['uuid'],
- ['clob', 'url', 'char', 'varchar']]
+# This list is used to determine the column to calculate the output size of 
the baskets. The list is in order of
+# preferences to make the calculation (from highest to lowest). We give 
preference to smaller types first, in order to
+# get more efficient computation (faster file reading).
+Types_Preferences = [['boolean', 'tinyint'], ['smallint'], ['int', 'real', 
'date', 'time', 'timetz', 'month_interval'],
+ ['bigint', 'sec_interval', 'double', 'timestamp', 
'timestamptz', 'inet'], ['decimal'], ['uuid'],
+ ['clob', 'url', 'char', 'varchar']]
 
 
 def polling_add_hugeint_type():
 Switcher.append({'types': ['hugeint'], 'class': HugeIntegerType})
-Queue[5].append('hugeint')
+Types_Preferences[5].append('hugeint')
 
 
 def init_stream_polling_thread(interval, connection):
@@ -84,7 +86,7 @@ def stream_polling(arguments):
 break
 
 for i in xrange(7):  # the queue has 7 entries
-if column[2] in Queue[i] and i < current_index:
+if column[2] in Types_Preferences[i] and i < 
current_index:
 current_index = i
 calc_column = new_column
 break
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Updated documentation, corrected some JSON schema...

[Pedro Ferreira]

Changeset: d18a2dfe460d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d18a2dfe460d
Modified Files:
clients/iotapi/documentation/websockets_api.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Updated documentation, corrected some JSON schemas listings and parsing


diffs (truncated from 322 to 300 lines):

diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -233,42 +233,44 @@ Returns a info message regarding all the
 "response": "data",
 "streams_count": 1,
 "streams_listing": [
-"schema": "measures",
-"stream": "temperature",
-"columns": [
-{
-"name": "sensorid",
-"type": "clob",
-"nullable": false,
-"default": "living room"
-},
-{
-"name": "temperature",
-"type": "real",
-"nullable": false,
-"default": null
-},
-{
-"name": "implicit_timestamp",
-"type": "timestamp with time zone",
-"nullable": false,
-"default": null
-}
-],
-"baskets_count": 3,
-"baskets_listing": [
-{
-"number": 1,
-"count": 20
-},
-{
-"number": 2,
-"count": 25
-},
-{
-"number": 3,
-"count": 12
-}
-]
+{
+"schema": "measures",
+"stream": "temperature",
+"columns": [
+{
+"name": "sensorid",
+"type": "clob",
+"nullable": false,
+"default": "living room"
+},
+{
+"name": "temperature",
+"type": "real",
+"nullable": false,
+"default": null
+},
+{
+"name": "implicit_timestamp",
+"type": "timestamp with time zone",
+"nullable": false,
+"default": null
+}
+],
+"baskets_count": 3,
+"baskets_listing": [
+{
+"number": 1,
+"count": 20
+},
+{
+"number": 2,
+"count": 25
+},
+{
+"number": 3,
+"count": 12
+}
+]
+}
 ]
 }
diff --git a/clients/iotclient/documentation/restful_resources.rst 
b/clients/iotclient/documentation/restful_resources.rst
--- a/clients/iotclient/documentation/restful_resources.rst
+++ b/clients/iotclient/documentation/restful_resources.rst
@@ -27,31 +27,31 @@ Returns a JSON file with details about a
{
"streams_count": 1,
 "streams_listing": [
-  {
-"schema": "measures",
-"stream": "temperature",
-"has_timestamp": true,
-"has_hostname": false,
-"columns": [
-{
-"name": "temperature",
-"type": "real",
-"default": null,
-"nullable": false
-},
-{
-"name": "sensorid",
-"type": "text",
-"default": null,
-"nullable": false
+{
+"schema": "measures",
+"stream": "temperature",
+"has_timestamp": true,
+"has_hostname": false,
+"columns": [
+{
+"name": "temperature",
+"type": "real",
+"default": null,
+"nullable": false
+},
+{
+"name": "sensorid",
+"type": "text",
+"default": null,
+"nullable": false
+}
+],
+"flushing": {
+"base": "tuple",

MonetDB: iot - Small corrections on code and documentation

[Pedro Ferreira]

Changeset: 2a0a5ab57c22 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2a0a5ab57c22
Added Files:
clients/iotclient/src/Utilities/customthreading.py
Modified Files:
clients/iotclient/documentation/index.rst
clients/iotclient/documentation/introduction.rst
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/flushing.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Small corrections on code and documentation


diffs (truncated from 583 to 300 lines):

diff --git a/clients/iotclient/documentation/index.rst 
b/clients/iotclient/documentation/index.rst
--- a/clients/iotclient/documentation/index.rst
+++ b/clients/iotclient/documentation/index.rst
@@ -18,4 +18,3 @@ Contents:
restful_resources.rst
streams_data_types.rst
conclusion.rst
-
diff --git a/clients/iotclient/documentation/introduction.rst 
b/clients/iotclient/documentation/introduction.rst
--- a/clients/iotclient/documentation/introduction.rst
+++ b/clients/iotclient/documentation/introduction.rst
@@ -1,14 +1,13 @@
 .. _instroduction:
 
-
 
 Introduction
 
 
-This appplication is a Python web server destinated to extend MonetDB RDBMS to 
the Internet-of-Things while using a streaming engine. This project is derived 
from Pedro Ferreira master thesis, a junior researcher at CWI. The objective of 
the master thesis to re-evaluate the DataCell extension of MonetDB to the IOT 
world, which was postponed for several years since its release. To accomplish 
this, an IOT topology around MonetDB is being built, where this server aims to 
analyse the input of streams. Later it will also be included another web server 
to analyze the output of streams.
+This application is a Python web server destined to extend MonetDB RDBMS to 
the Internet-of-Things while using a streaming engine. This project is derived 
from Pedro Ferreira master thesis, a junior researcher at CWI. The objective of 
the master thesis to re-evaluate the DataCell extension of MonetDB to the IOT 
world, which was postponed for several years since its release. To accomplish 
this, an IOT topology around MonetDB is being built, where this server aims to 
analyse the input of streams. Later it will also be included another web server 
to analyze the output of streams.
 
 This web server is built using Python programming language, version 2.7, using 
`Flask-RESTful <https://pypi.python.org/pypi/Flask-RESTful>`_ framework. The 
required packages on requirements.txt should be installed before running the 
server. 
 
-The web server is capable of creating and deleting streams for the renewed 
DataCell extension. After a stream is created, it's possible to make batch 
inserts on it. The requests are made using RESTfull requests with JSON content. 
Both stream creation and stream insertion are validated using `JSON Schema 
Draft 4 <http://json-schema.org/documentation.html>`_. On a batch insert if a 
tuple is invalid, then no tuples are inserted. The stream engine adds an 
implicit timestamp column to recognize when the tuple was inserted. The 
inserted tuples are later evaluated using MonetDB's relational engine with 
continuous queries.
+The web server is capable of creating and deleting streams for the renewed 
DataCell extension. After a stream is created, it's possible to make batch 
inserts on it. The requests are made using RESTful requests with JSON content. 
Both stream creation and stream insertion are validated using `JSON Schema 
Draft 4 <http://json-schema.org/documentation.html>`_. On a batch insert if a 
tuple is invalid, then no tuples are inserted. The stream engine adds an 
implicit timestamp column to recognize when the tuple was inserted. The 
inserted tuples are later evaluated using MonetDB's relational engine with 
continuous queries.
 
 The following chapters explain the server parameters, RESTful resources and 
how to manage the streams.
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -1,11 +1,12 @@
 .. _starting_webserver:
 
-
 ***
 Starting the web server
 ***
 
-The server is started as a regular Python program invoking the :code:`main.py` 
file, however ther are several arguments that can b

MonetDB: iot - Fixed several bugs

[Pedro Ferreira]

Changeset: 2a30799892db for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2a30799892db
Modified Files:
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Fixed several bugs


diffs (truncated from 427 to 300 lines):

diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -104,7 +104,6 @@ class StreamDataType(object):
 def create_stream_sql(self):  # get column creation statement on SQL
 array = [self._column_name, " ", self._data_type]
 self.process_sql_parameters(array)  # add extra parameters to the SQL 
statement
-
 if self._default_value is not None:
 array.extend([" DEFAULT '", str(self._default_value), "'"])
 if not self._is_nullable:
@@ -237,11 +236,11 @@ class RegexType(TextType):
 
 def add_json_schema_entry(self, schema):
 super(RegexType, self).add_json_schema_entry(schema)
-schema[self._column_name]['pattern'] = self._regex
+schema[self._column_name]['pattern'] = self._regex.pattern
 
 def to_json_representation(self):
 json_value = super(RegexType, self).to_json_representation()
-json_value['regex'] = self._regex
+json_value['regex'] = self._regex.pattern
 return json_value
 
 def process_sql_parameters(self, array):
@@ -253,7 +252,7 @@ class LimitedTextType(TextType):
 
 def __init__(self, **kwargs):
 super(LimitedTextType, self).__init__(**kwargs)
-self._limit = int(kwargs['limit'])
+self._limit = kwargs['limit']
 
 def add_json_schema_entry(self, schema):
 super(LimitedTextType, self).add_json_schema_entry(schema)
@@ -263,7 +262,8 @@ class LimitedTextType(TextType):
 str_value = str(default_value)
 parsed_len = len(str_value)
 if parsed_len > self._limit:
-raise Exception('The default string is higher than the limit: %d > 
%d' % (parsed_len, self._limit))
+raise Exception('The default string\'s length is longer than the 
limit: %d > %d!'
+% (parsed_len, self._limit))
 self._default_value = str_value
 
 def to_json_representation(self):
@@ -340,9 +340,9 @@ class NumberBaseType(StreamDataType):
 def __init__(self, **kwargs):
 super(NumberBaseType, self).__init__(**kwargs)
 if 'minimum' in kwargs:
-self._minimum = self.process_next_value(kwargs['minimum'], 0, {}, 
{})
+self._minimum = kwargs['minimum']
 if 'maximum' in kwargs:
-self._maximum = self.process_next_value(kwargs['maximum'], 0, {}, 
{})
+self._maximum = kwargs['maximum']
 if hasattr(self, '_minimum') and hasattr(self, '_maximum') and 
self._minimum > self._maximum:
 raise Exception('The minimum value is higher than the maximum!')
 
@@ -353,21 +353,16 @@ class NumberBaseType(StreamDataType):
 if hasattr(self, '_maximum'):
 schema[self._column_name]['maximum'] = self._maximum
 
-@abstractmethod
-def process_default_value(self, value):
-return value
-
 def set_default_value(self, default_value):
-parsed_val = self.process_default_value(default_value)
-if hasattr(self, '_minimum') and not hasattr(self, '_maximum') and 
parsed_val < self._minimum:
-raise Exception('The default value is less than the minimum: %d < 
%d' % (parsed_val, self._minimum))
-elif hasattr(self, '_maximum') and not hasattr(self, '_minimum') and 
parsed_val > self._maximum:
-raise Exception('The default value is higher than the maximum: %d 
> %d' % (parsed_val, self._maximum))
-elif hasattr(self, '_maximum') and hasattr(self, '_minimum') and 
parsed_val < self._minimum:
-raise Exception('The default value is out of range: %d < %d' % 
(parsed_val, self._minimum))
-elif hasattr(self, '_maximum') and hasattr(self, '_minimum') and 
parsed_val > self._maximum:
-raise Exception('The default value is out of range: %d > %d' % 
(parsed_val, self._maximum))
-self._default_value = parsed_val
+if hasattr(self, '_minimum') and not hasattr(self, '_maximum') and 
default_value < self._minimum:
+raise Exception('The default value is less than the minimum: %s < 
%s!' % (default_value, self._minimum))
+elif hasattr(self, '_maximum') and not hasattr(self, '_minimum') and 
default_value > self._maximum:
+raise Exception('The default value is higher than the maximum: %s 
> %s!' % (default_value, self._maximum))
+elif hasattr(self, '_maximum') and hasattr(self, '_minimum') and 
default_value < self._minimum:
+

MonetDB: iot - More bugfixes

[Pedro Ferreira]

Changeset: e13cdaed7ea2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e13cdaed7ea2
Modified Files:
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

More bugfixes


diffs (202 lines):

diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -406,7 +406,7 @@ class SmallIntegerType(NumberBaseType):
 return int(entry)
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
-return struct.pack(ALIGNMENT + str(counter) + self._pack_sym, 
extracted_values)
+return struct.pack(ALIGNMENT + str(counter) + self._pack_sym, 
*extracted_values)
 
 
 class HugeIntegerType(NumberBaseType):
@@ -540,7 +540,9 @@ class BaseDateTimeType(StreamDataType): 
 self._maximum = self.parse_entry(kwargs['maximum'])
 if hasattr(self, '_minimum') and hasattr(self, '_maximum') and 
self._minimum > self._maximum:
 raise Exception('The minimum value is higher than the maximum!')
-self._default_value_text = None  # needed later for the SQL creation 
statement
+
+def get_nullable_constant(self):
+return "0"
 
 @abstractmethod
 def parse_entry(self, entry):
@@ -558,14 +560,15 @@ class BaseDateTimeType(StreamDataType): 
 raise Exception('The default value is out of range: %s < %s!' % 
(default_value, self._minimum_text))
 elif hasattr(self, '_maximum') and hasattr(self, '_minimum') and 
parsed_val > self._maximum:
 raise Exception('The default value is out of range: %s > %s!' % 
(default_value, self._maximum_text))
-self._default_value = parsed_val
-self._default_value_text = default_value
+self._default_value = default_value
 
 @abstractmethod
 def pack_next_value(self, parsed, counter, parameters, errors):
 pass
 
 def process_next_value(self, entry, counter, parameters, errors):
+if entry == self.get_nullable_constant():  # have to do this trick due 
to Python's datetime limitations
+return self.pack_next_value(None, counter, parameters, errors)
 parsed = self.parse_entry(entry)
 if hasattr(self, '_minimum') and not hasattr(self, '_maximum') and 
parsed < self._minimum:
 errors[counter] = 'The value is higher than the minimum: %s < %s!' 
% (self._minimum_text, parsed)
@@ -577,31 +580,16 @@ class BaseDateTimeType(StreamDataType): 
 errors[counter] = 'The value is out of range: %s > %s!' % (parsed, 
self._maximum_text)
 return self.pack_next_value(parsed, counter, parameters, errors)
 
-def add_json_schema_entry(self, schema):
-super(BaseDateTimeType, self).add_json_schema_entry(schema)
-schema[self._column_name]['default'] = self._default_value_text
-
 def to_json_representation(self):
 json_value = super(BaseDateTimeType, self).to_json_representation()
-json_value['type'] = 'string'
-if self._default_value is not None:
-json_value['default'] = self._default_value_text
 if hasattr(self, '_minimum'):
 json_value['minimum'] = self._minimum_text
 if hasattr(self, '_maximum'):
 json_value['maximum'] = self._maximum_text
 return json_value
 
-def create_stream_sql(self):
-array = [self._column_name, " ", self._data_type]
-if self._default_value is not None:
-array.extend([" DEFAULT '", str(self._default_value_text), "'"])
-if not self._is_nullable:
-array.append(" NOT NULL")
-return ''.join(array)
 
-
-class DateType(BaseDateTimeType):  # Stored as an uint with the number of days 
since day 0 of month 1 (Jan) from year 0
+class DateType(BaseDateTimeType):  # Stored as an uint with the number of days 
since day 1 of month 1 (Jan) from year 0
 """Covers: DATE"""
 
 def __init__(self, **kwargs):
@@ -611,15 +599,15 @@ class DateType(BaseDateTimeType):  # Sto
 super(DateType, self).add_json_schema_entry(schema)
 schema[self._column_name]['format'] = 'date'
 
-def get_nullable_constant(self):
-return INT32_MIN  # Checked from MonetDB's source code
-
 def parse_entry(self, entry):
 return datetime.datetime.strptime(str(entry), "%Y-%m-%d")
 
 def pack_next_value(self, parsed, counter, parameters, errors):
+if parsed is None:
+return INT32_MIN
 day0 = copy.deepcopy(parsed).replace(year=1, month=1, day=1)
-return (parsed - day0).days + 366  # the mindate in python is 1, but 
for the representation is 0, so why the add
+# the minyear in python is 1, but for the representation is 0, so why 
the add
+return int((parsed - day0).days) + 366
 
 def pack_parsed_values(self, extracted_values, 

MonetDB: iot - Made important corrections

[Pedro Ferreira]

Changeset: dae88d156163 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dae88d156163
Modified Files:
clients/iotclient/documentation/conf.py
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Made important corrections


diffs (truncated from 302 to 300 lines):

diff --git a/clients/iotclient/documentation/conf.py 
b/clients/iotclient/documentation/conf.py
--- a/clients/iotclient/documentation/conf.py
+++ b/clients/iotclient/documentation/conf.py
@@ -12,8 +12,8 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
-import os
+# import sys
+# import os
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -110,7 +110,7 @@ pygments_style = 'sphinx'
 # If true, keep warnings as "system message" paragraphs in the built documents.
 #keep_warnings = False
 
-# If true, `todo` and `todoList` produce output, else they produce nothing.
+# If true, `ttodo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
 
diff --git a/clients/iotclient/src/Flask/app.py 
b/clients/iotclient/src/Flask/app.py
--- a/clients/iotclient/src/Flask/app.py
+++ b/clients/iotclient/src/Flask/app.py
@@ -1,6 +1,5 @@
 from flask import Flask
 from flask_restful import Api
-
 from restresources import StreamInput, StreamsInfo, StreamsHandling
 
 
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -8,38 +8,37 @@ from flask_restful import Resource
 from jsonschema import Draft4Validator, FormatChecker
 from tzlocal import get_localzone
 from Streams.jsonschemas import CREATE_STREAMS_SCHEMA, DELETE_STREAMS_SCHEMA
-from Streams.streamscontext import IOTStreamsException, IOTStreams
+from Streams.streamscontext import IOTStreams
 from Settings.iotlogger import add_log
 
 Streams_Context = None
 Create_Streams_Validator = None
 Delete_Streams_Validator = None
-Local_Timezone = None
+Local_Timezone = get_localzone()  # for the correction of dates we must add 
the system's timezone
 
 
 def init_rest_resources():
 global Streams_Context, Create_Streams_Validator, 
Delete_Streams_Validator, Local_Timezone
 
-Local_Timezone = get_localzone()  # for the correction of dates we must 
add the system's timezone
 Create_Streams_Validator = Draft4Validator(CREATE_STREAMS_SCHEMA, 
format_checker=FormatChecker())
 Delete_Streams_Validator = Draft4Validator(DELETE_STREAMS_SCHEMA, 
format_checker=FormatChecker())
 try:
 Streams_Context = IOTStreams()
 except BaseException as ex:
 print >> sys.stdout, ex
-add_log(50, ex.message)
+add_log(50, ex)
 sys.exit(1)
 
 
-class StreamInput(Resource):  # TODO these operations are not atomic!!!
+class StreamInput(Resource):
 """RESTful API for stream's input"""
 
 def get(self, schema_name, stream_name):  # check a single stream data
 try:  # check if stream exists, if not return 404
 stream = Streams_Context.get_existing_stream(schema_name, 
stream_name)
 except BaseException as ex:
-add_log(50, ex.message)
-return ex.message, 404
+add_log(50, ex)
+return ex, 404
 return stream.get_data_dictionary(include_number_tuples=True), 200
 
 def post(self, schema_name, stream_name):  # add data to a stream
@@ -48,14 +47,14 @@ class StreamInput(Resource):  # TODO the
 try:  # check if stream exists, if not return 404
 stream = Streams_Context.get_existing_stream(schema_name, 
stream_name)
 except BaseException as ex:
-add_log(50, ex.message)
-return ex.message, 404
+add_log(50, ex)
+return ex, 404
 
 try:  # validate and insert data, if not return 400
 stream.validate_and_insert(json.loads(request.data), current_stamp)
 except BaseException as ex:
-add_log(50, ex.message)
-return ex.message, 400
+add_log(50, ex)
+return ex, 400
 return 'The insertions were made with success!', 201
 
 
@@ -78,8 +77,8 @@ class StreamsHandling(Resource):
 Create_Streams_Validator.validate(schema_to_validate)
 Streams_Context.add_new_stream(schema_to_validate)
 except BaseException as ex:
-add_log(50, ex.message)
-return ex.message, 400
+add_log(50, ex)
+ 

MonetDB: iot - SIGINT signal working properly. Created subproces...

[Pedro Ferreira]

Changeset: 9d8064a14b93 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9d8064a14b93
Modified Files:
clients/iotclient/src/main.py
Branch: iot
Log Message:

SIGINT signal working properly. Created subprocess for each server


diffs (61 lines):

diff --git a/clients/iotclient/src/main.py b/clients/iotclient/src/main.py
--- a/clients/iotclient/src/main.py
+++ b/clients/iotclient/src/main.py
@@ -2,6 +2,7 @@ import getopt
 import signal
 import sys
 
+from multiprocessing import Process
 from uuid import getnode as get_mac
 from Flask.app import start_flask_iot_app, start_flask_admin_app
 from Flask.restresources import init_rest_resources
@@ -10,22 +11,18 @@ from Settings.iotlogger import init_logg
 from Settings.mapiconnection import init_monetdb_connection
 from Streams.streamscontext import init_streams_context
 from Streams.streams import init_streams_hosts
-from Utilities.customthreading import StoppableThread
 
-thread1 = None
-thread2 = None
+subprocess1 = None
+subprocess2 = None
 
 
 def signal_handler(signal, frame):
-print >> sys.stdout, 'Kill me!!'
-thread1.stop()
-thread2.stop()
-add_log(20, 'Stopped IOT Stream Server')
-sys.exit(0)
+subprocess1.terminate()
+subprocess2.terminate()
 
 
 def main(argv):
-global thread1, thread2
+global subprocess1, subprocess2
 
 try:
 opts, args = getopt.getopt(argv[1:], 'f:l:c:ui:in:ih:ip:ah:ap:h:p:d:u',
@@ -95,15 +92,15 @@ def main(argv):
 init_streams_context()  # init streams context
 init_rest_resources()  # init validators for RESTful requests
 
-thread1 = StoppableThread(target=start_flask_admin_app, args=(admin_host, 
admin_port))
-thread2 = StoppableThread(target=start_flask_iot_app, args=(app_host, 
app_port))
-thread1.start()
-thread2.start()
+subprocess1 = Process(target=start_flask_admin_app, args=(admin_host, 
admin_port))
+subprocess2 = Process(target=start_flask_iot_app, args=(app_host, 
app_port))
+subprocess1.start()
+subprocess2.start()
 add_log(20, 'Started IOT Stream Server')
 signal.signal(signal.SIGINT, signal_handler)
 
-thread1.join()
-thread2.join()
+subprocess1.join()
+subprocess2.join()
 add_log(20, 'Stopped IOT Stream Server')
 
 if __name__ == "__main__":
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Several bugfixes

[Pedro Ferreira]

Changeset: 82ab5c86a247 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=82ab5c86a247
Modified Files:
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Several bugfixes


diffs (truncated from 301 to 300 lines):

diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -33,7 +33,7 @@ def fetch_streams():
 try:  # TODO paginate results?
 cursor = Connection.cursor()
 sql_string = """SELECT storage."schema", storage."table", 
storage."column", storage."type", storage."typewidth"
-  FROM (SELECT "schema", "table", "column", "type" FROM sys.storage) 
AS storage
+  FROM (SELECT "schema", "table", "column", "type", "typewidth" FROM 
sys.storage) AS storage
   INNER JOIN (SELECT "name" FROM sys.tables WHERE type=4) AS tables ON 
(storage."table"=tables."name")
   INNER JOIN (SELECT "name" FROM sys.schemas) AS schemas ON 
(storage."schema"=schemas."name");"""\
 .replace('\n', ' ')
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -4,7 +4,7 @@ import struct
 from datatypes import LITTLE_ENDIAN_ALIGNMENT
 from Settings.filesystem import get_baskets_base_location
 from Utilities.readwritelock import RWLock
-from WebSockets.websockets import notify_clients
+from WebSockets.websockets import notify_stream_inserts_to_clients
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
@@ -30,7 +30,7 @@ class StreamBasketsHandler(FileSystemEve
 if isinstance(event, 'DirCreatedEvent'):
 basket_string = os.path.basename(os.path.normpath(event.src_path))
 count = self._stream.append_basket(basket_string)
-notify_clients(self._stream.get_schema_name(), 
self._stream.get_stream_name(), count)
+notify_stream_inserts_to_clients(self._stream.get_schema_name(), 
self._stream.get_stream_name(), count)
 
 def on_deleted(self, event):
 if isinstance(event, 'DirDeletedEvent'):
diff --git a/clients/iotapi/src/WebSockets/websockets.py 
b/clients/iotapi/src/WebSockets/websockets.py
--- a/clients/iotapi/src/WebSockets/websockets.py
+++ b/clients/iotapi/src/WebSockets/websockets.py
@@ -3,7 +3,6 @@ import sys
 
 from Settings.iotlogger import add_log
 from SimpleWebSocketServer import SimpleWebSocketServer, WebSocket
-from Streams.streamscontext import Streams_context, IOTStreams
 from Utilities.readwritelock import RWLock
 from jsonschema import Draft4Validator, FormatChecker
 
@@ -15,14 +14,6 @@ WebClients = []  # TODO this probably wo
 WebClientsLock = RWLock()
 
 
-def notify_stream_inserts_to_clients(schema_name, stream_name, count):
-concatenated_name = IOTStreams.get_context_entry_name(schema_name, 
stream_name)
-WebClientsLock.acquire_read()
-for client in WebClients:
-client.send_notification_message(concatenated_name, schema_name, 
stream_name, count)
-WebClientsLock.release()
-
-
 def unsubscribe_removed_streams(concatenated_names):
 WebClientsLock.acquire_read()
 for name in concatenated_names:
@@ -32,6 +23,16 @@ def unsubscribe_removed_streams(concaten
 for name in concatenated_names:
 add_log(20, ''.join(['Stream ', name, ' removed']))
 
+from Streams.streamscontext import Streams_context, IOTStreams  # avoid 
circular dependency
+
+
+def notify_stream_inserts_to_clients(schema_name, stream_name, count):
+concatenated_name = IOTStreams.get_context_entry_name(schema_name, 
stream_name)
+WebClientsLock.acquire_read()
+for client in WebClients:
+client.send_notification_message(concatenated_name, schema_name, 
stream_name, count)
+WebClientsLock.release()
+
 
 class IOTAPI(WebSocket):
 """Client WebSocket"""
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -8,7 +8,7 @@ from flask_restful import Resource
 from jsonschema import Draft4Validator, FormatChecker
 from tzlocal import get_localzone
 from Streams.jsonschemas import CREATE_STREAMS_SCHEMA, DELETE_STREAMS_SCHEMA
-from Streams.streamscontext import IOTStreams
+from Streams.streamscontext import IOTStreamsContext
 from Settings.iotlogger import add_log

MonetDB: iot - Cleaned code

[Pedro Ferreira]

Changeset: ee944c8e9c97 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ee944c8e9c97
Removed Files:
clients/iotclient/src/Streams/flushing.py
Modified Files:
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/WebSockets/jsonschemas.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/filecreator.py
Branch: iot
Log Message:

Cleaned code


diffs (truncated from 578 to 300 lines):

diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -12,9 +12,6 @@ def init_monetdb_connection(hostname, po
 
 user_password = getpass.getpass(prompt='Insert password for user ' + 
user_name + ':')
 
-if user_password == '':
-user_password = 'monetdb'
-
 try:  # the autocommit is set to true so each statement will be independent
 Connection = pymonetdb.connect(hostname=hostname, port=port, 
username=user_name, password=user_password,
database=database, autocommit=True)
@@ -35,15 +32,12 @@ def close_monetdb_connection():
 def fetch_streams():
 try:  # TODO paginate results?
 cursor = Connection.cursor()
-sql_string = """
-  SELECT storage."schema", storage."table", storage."column", 
storage."type", storage."location",
-  storage."typewidth"
+sql_string = """SELECT storage."schema", storage."table", 
storage."column", storage."type", storage."typewidth"
   FROM (SELECT "schema", "table", "column", "type" FROM sys.storage) 
AS storage
   INNER JOIN (SELECT "name" FROM sys.tables WHERE type=4) AS tables ON 
(storage."table"=tables."name")
-  INNER JOIN (SELECT "name" FROM sys.schemas) AS schemas ON 
(storage."schema"=schemas."name");
-""".replace('\n', ' ')
+  INNER JOIN (SELECT "name" FROM sys.schemas) AS schemas ON 
(storage."schema"=schemas."name");"""\
+.replace('\n', ' ')
 cursor.execute(sql_string)
 return cursor.fetchall()
 except BaseException as ex:
-print >> sys.stdout, ex
 add_log(50, ex)
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -2,8 +2,9 @@ from itertools import groupby
 
 from Settings.mapiconnection import fetch_streams
 from Utilities.customthreading import PeriodicalThread
-
-from streamscontext import Streams_context, DataCellStream
+from datatypes import *
+from streams import IOTStream
+from streamscontext import Streams_context
 
 SWITCHER = [{'types': ['clob', 'char', 'varchar', 'url'], 'class': 'TextType'},
 {'types': ['tinyint', 'smallint', 'int', 'bigint'], 'class': 
'SmallIntegerType'},
@@ -23,7 +24,7 @@ def init_stream_polling_thread(interval)
 thread.start()
 
 
-# elem[0] is schema. elem[1] is name, elem[2] is column name, elem[3] is type, 
elem[4] is location, elem[5] is typewidth
+# elem[0] is schema. elem[1] is name, elem[2] is column name, elem[3] is type, 
elem[4] is typewidth
 def stream_polling():
 current_streams = Streams_context.get_existing_streams()
 retained_streams = []
@@ -35,11 +36,13 @@ def stream_polling():
 columns = {}
 
 for elem in group:
-reflection_class = globals()[elem[3]]  # import everything 
from datatypes!!!
-kwargs = {'name': elem[2], 'type': elem[3], 'location': 
elem[4], 'typewidth': elem[5]}
-new_column = reflection_class(kwargs)
-columns[elem[2]] = new_column
-new_streams[key] = DataCellStream(key, columns)
+for entry in SWITCHER:  # allocate the proper type wrapper
+if elem[3] in entry['types']:
+reflection_class = globals()[entry['class']]  # import 
everything from datatypes!!!
+new_column = reflection_class({'name': elem[2], 
'type': elem[3], 'typewidth': elem[4]})
+columns[elem[2]] = new_column
+new_streams[key] = IOTStream(key, columns)
+break
 else:
 retained_streams.append(key)
 
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -1,6 +1,7 @@
 import os
 import 

MonetDB: iot - Small corrections

[Pedro Ferreira]

Changeset: ece85c5d8d31 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ece85c5d8d31
Modified Files:
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Streams/datatypes.py
Branch: iot
Log Message:

Small corrections


diffs (104 lines):

diff --git a/clients/iotapi/src/WebSockets/websockets.py 
b/clients/iotapi/src/WebSockets/websockets.py
--- a/clients/iotapi/src/WebSockets/websockets.py
+++ b/clients/iotapi/src/WebSockets/websockets.py
@@ -30,7 +30,7 @@ def unsubscribe_removed_streams(concaten
 client.remove_subscribed_stream(name)
 WebClientsLock.release()
 for name in concatenated_names:
-add_log(20, ''.join(['Stream removed: ', name]))
+add_log(20, ''.join(['Stream ', name, ' removed']))
 
 
 class IOTAPI(WebSocket):
@@ -81,7 +81,7 @@ class IOTAPI(WebSocket):
 self._subscriptions[concatenated_name] = stream
 self._locker.release()
 self.sendMessage({"subscribed": "Subscribed to " + concatenated_name})
-add_log(20, ''.join(['Client ', self.address[0], 'subscribed stream ', 
concatenated_name]))
+add_log(20, ''.join(['Client ', self.address[0], 'subscribed to stream 
', concatenated_name]))
 
 def unsubscribe(self, concatenated_name):
 self._locker.acquire_write()
@@ -92,27 +92,31 @@ class IOTAPI(WebSocket):
 del self._subscriptions[concatenated_name]
 self._locker.release()
 self.sendMessage({"unsubscribed": "Unsubscribed to " + 
concatenated_name})
-add_log(20, ''.join(['Client ', self.address[0], ' unsubscribed 
stream ', concatenated_name]))
+add_log(20, ''.join(['Client ', self.address[0], ' unsubscribed to 
stream ', concatenated_name]))
 
 def remove_subscribed_stream(self, concatenated_name):
 self._locker.acquire_write()
 if concatenated_name in self._subscriptions:
 del self._subscriptions[concatenated_name]
 self._locker.release()
-self.sendMessage({"removed": "Stream removed from context: " + 
concatenated_name})
+self.sendMessage({"removed": 'Stream ' + concatenated_name + ' removed 
from context'})
 
 def send_notification_message(self, concatenated_name, schema_name, 
stream_name, count):
 self._locker.acquire_read()
 if concatenated_name in self._subscriptions:
 self._locker.release()
 self.sendMessage({'notification': {'schema': schema_name, 
'stream': stream_name, 'tuples': count}})
-add_log(20, ''.join(['Stream notification sent to client ', 
self.address[0]]))
+add_log(20, ''.join(['Stream ', concatenated_name, ' notification 
sent to client ', self.address[0]]))
 else:
 self._locker.release()
 
 def read_stream_batch(self, concatenated_name, basket_number, limit, 
offset):
-stream = Streams_context.get_existing_stream(concatenated_name)
-self.sendMessage(stream.read_tuples(basket_number, limit, offset))
+try:
+stream = Streams_context.get_existing_stream(concatenated_name)
+self.sendMessage(stream.read_tuples(basket_number, limit, offset))
+except BaseException as ex:
+self.sendMessage({"error": ex})
+add_log(50, ex)
 
 
 def init_websockets(host, port):
diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -288,16 +288,14 @@ class INetType(StreamDataType):
 
 if entry == self.get_nullable_constant():
 array[7] = 1
-return array
-
-components = re.split(r'[./]+', entry)
-for i in range(4):
-array[i] = int(components[i])
-if len(components) > 4:  # if it has a mask add it to the array
-array[4] = int(components[4])
 else:
-array[4] = 32
-
+components = re.split(r'[./]+', entry)
+for i in xrange(4):
+array[i] = int(components[i])
+if len(components) > 4:  # if it has a mask add it to the array
+array[4] = int(components[4])
+else:
+array[4] = 32
 return array
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
@@ -323,7 +321,7 @@ class UUIDType(StreamDataType):
 j = 0
 s = 0
 
-for i in range(16):
+for i in xrange(16):
 if j in (8, 12, 16, 20):  # do nothing with the dashes
 s += 1
 
@@ -338,8 +336,8 @@ class UUIDType(StreamDataType):
 s += 1
 j += 1
 array[i] <<= 4
+
 next_char = ord(entry[s])
-
 if 48 <= next_char <= 57:  # between '0' and '9'
 array[i] |= next_char - 48
 elif 97 <= next_char <= 102:  # between 'a' and 'f'

MonetDB: iot - Created subprocess for iot api

[Pedro Ferreira]

Changeset: dde8adbf4d2a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dde8adbf4d2a
Modified Files:
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotapi/src/main.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Created subprocess for iot api


diffs (147 lines):

diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -7,11 +7,9 @@ from Settings.iotlogger import add_log
 Connection = None
 
 
-def init_monetdb_connection(hostname, port, user_name, database):
+def init_monetdb_connection(hostname, port, user_name, user_password, 
database):
 global Connection
 
-user_password = getpass.getpass(prompt='Insert password for user ' + 
user_name + ':')
-
 try:  # the autocommit is set to true so each statement will be independent
 Connection = pymonetdb.connect(hostname=hostname, port=port, 
username=user_name, password=user_password,
database=database, autocommit=True)
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -39,9 +39,9 @@ def stream_polling():
 for entry in SWITCHER:  # allocate the proper type wrapper
 if elem[3] in entry['types']:
 reflection_class = globals()[entry['class']]  # import 
everything from datatypes!!!
-new_column = reflection_class({'name': elem[2], 
'type': elem[3], 'typewidth': elem[4]})
+new_column = reflection_class(**{'name': elem[2], 
'type': elem[3], 'typewidth': elem[4]})
 columns[elem[2]] = new_column
-new_streams[key] = IOTStream(key, columns)
+new_streams[key] = IOTStream(elem[0], elem[1], 
**columns)
 break
 else:
 retained_streams.append(key)
diff --git a/clients/iotapi/src/Utilities/customthreading.py 
b/clients/iotapi/src/Utilities/customthreading.py
--- a/clients/iotapi/src/Utilities/customthreading.py
+++ b/clients/iotapi/src/Utilities/customthreading.py
@@ -20,13 +20,12 @@ class StoppableThread(Thread):
 class PeriodicalThread(StoppableThread):
 """Thread working with a timed interval basis"""
 
-def __init__(self, interval, worker_func, func_args=None, *args, **kwargs):
+def __init__(self, interval, worker_func, *args, **kwargs):
 super(PeriodicalThread, self).__init__(*args, **kwargs)
 self._interval = interval  # in seconds
 self._worker_func = worker_func  # function/method to execute 
periodically
-self._worker_func_args = func_args
 
 def run(self):
 while not self.stop_event.is_set():
-self._worker_func(self._worker_func_args)
+self._worker_func()
 time.sleep(self._interval)
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -1,19 +1,46 @@
 import getopt
+import getpass
 import signal
 import sys
 
+from multiprocessing import Process
+from threading import Thread
 from Settings.filesystem import init_file_system, set_filesystem_location
 from Settings.iotlogger import init_logging, add_log, set_logging_location
-from Settings.mapiconnection import init_monetdb_connection
+from Settings.mapiconnection import init_monetdb_connection, 
close_monetdb_connection
 from Streams.streampolling import init_stream_polling_thread
 from WebSockets.websockets import init_websockets, terminate_websockets
 
+subprocess = None
 
-def close_sig_handler(signal, frame):
+
+def signal_handler(signal, frame):
+subprocess.terminate()
+
+
+def start_process(sockets_host, sockets_port, connection_hostname, 
connection_port, connection_user,
+  connection_password, connection_database):
+# WARNING The initiation order must be this!!!
+init_logging()  # init logging context
+init_file_system()  # init filesystem
+# init mapi connection
+init_monetdb_connection(connection_hostname, connection_port, 
connection_user, connection_password,
+connection_database)
+init_stream_polling_thread(60)  # start polling
+
+thread1 = Thread(target=init_websockets, args=(sockets_host, sockets_port))
+thread1.start()
+add_log(20, 'Started IOT API Server')
+thread1.join()
+
 terminate_websockets()
+close_monetdb_connection()
+add_log(20, 'Stopped IOT API Server')
 
 
 def main(argv):
+global subprocess
+
 try:
 opts, args = getopt.getopt(argv[1:], 

MonetDB: iot - Added auto-flushed streams, removed unnecessary v...

[Pedro Ferreira]

Changeset: f9baa0c79960 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f9baa0c79960
Modified Files:
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/main.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Added auto-flushed streams, removed unnecessary variables, web api is now 
running as a subprocess


diffs (truncated from 780 to 300 lines):

diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -1,34 +1,26 @@
 import sys
-
 import os
 
 from iotlogger import add_log
 
-BASKETS_BASE_DIRECTORY = "baskets"
-
-if sys.platform in ("linux", "linux2", "darwin"):
-Filesystem_Location = '/etc/iotcollector'
-elif sys.platform == "win32":
-Filesystem_Location = os.path.join(os.path.dirname(__file__), os.pardir)
-
 Baskets_Location = None
 
 
-def set_filesystem_location(new_location):
-global Filesystem_Location
-Filesystem_Location = new_location
-
-
-def init_file_system():
+def init_file_system(new_location=None):
 global Baskets_Location
 
+if new_location is None:
+if sys.platform in ("linux", "linux2", "darwin"):
+new_location = '/etc/iotcollector'
+elif sys.platform == "win32":
+new_location = os.path.join(os.path.dirname(__file__), os.pardir)
+else:
+new_location = new_location
+
 try:
-Baskets_Location = os.path.join(Filesystem_Location, 
BASKETS_BASE_DIRECTORY)
+Baskets_Location = os.path.join(new_location, "baskets")
 if not os.path.exists(Baskets_Location):
 os.makedirs(Baskets_Location)
-
-if not os.path.exists(Filesystem_Location):
-os.makedirs(Filesystem_Location)
 except (Exception, OSError) as ex:
 print >> sys.stdout, ex
 add_log(50, ex)
diff --git a/clients/iotapi/src/Settings/iotlogger.py 
b/clients/iotapi/src/Settings/iotlogger.py
--- a/clients/iotapi/src/Settings/iotlogger.py
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -3,23 +3,22 @@ import sys
 
 import os
 
-if sys.platform in ("linux", "linux2", "darwin"):
-logging_location = '/var/log/iot/iotapi.log'
-elif sys.platform == "win32":
-logging_location = os.path.join(os.path.dirname(__file__), os.pardir, 
'iotapi.log')
+Logger = logging.getLogger("IOTAPILog")
 
-logger = logging.getLogger("IOTAPILog")
 
+def init_logging(new_location):
+global Logger
 
-def set_logging_location(new_location):
-global logging_location
-logging_location = new_location
+if new_location is None:
+if sys.platform in ("linux", "linux2", "darwin"):
+logging_location = '/var/log/iot/iotapi.log'
+elif sys.platform == "win32":
+logging_location = os.path.join(os.path.dirname(__file__), 
os.pardir, 'iotapi.log')
+else:
+logging_location = new_location
 
-
-def init_logging():
-global logger
 try:
-logger = logging.getLogger("IOTLog")
+logger = logging.getLogger("IOTAPILog")
 logger.setLevel(logging.DEBUG)
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s 
- %(message)s')
 
@@ -35,4 +34,4 @@ def init_logging():
 
 
 def add_log(lvl, message, *args, **kwargs):
-logger.log(lvl, message, *args, **kwargs)
+Logger.log(lvl, message, *args, **kwargs)
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -5,8 +5,8 @@ import sys
 
 from multiprocessing import Process
 from threading import Thread
-from Settings.filesystem import init_file_system, set_filesystem_location
-from Settings.iotlogger import init_logging, add_log, set_logging_location
+from Settings.filesystem import init_file_system
+from Settings.iotlogger import init_logging, add_log
 from Settings.mapiconnection import init_monetdb_connection, 
close_monetdb_connection
 from Streams.streampolling import init_stream_polling_thread
 from WebSockets.websockets import init_websockets, terminate_websockets
@@ -18,14 +18,13 @@ def signal_handler(signal, frame):
 subprocess.terminate()
 
 
-def start_process(sockets_host, sockets_port, connection_hostname, 
connection_port, connection_user,
-  connection_password, connection_database):
+def start_process(filesystem_location, logging_location, sockets_host, 
sockets_port, connection_hostname, 

MonetDB: iot - Started to work on Web API for iot

[Pedro Ferreira]

Changeset: 0718a6157981 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0718a6157981
Added Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Settings/__init__.py
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/__init__.py
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/__init__.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotapi/src/__init__.py
clients/iotapi/src/main.py
Modified Files:
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/flushing.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/customthreading.py
Branch: iot
Log Message:

Started to work on Web API for iot


diffs (truncated from 770 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/requirements.txt
@@ -0,0 +1,7 @@
+git+https://github.com/dpallot/simple-websocket-server.git
+python-dateutil==2.5.3
+pytz==2016.4
+pymonetdb==0.1.1
+tzlocal==1.2.2
+Sphinx==1.4.1
+sphinx-rtd-theme==0.1.9
diff --git a/clients/iotapi/src/Settings/__init__.py 
b/clients/iotapi/src/Settings/__init__.py
new file mode 100644
diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -0,0 +1,28 @@
+import sys
+
+import os
+
+from iotlogger import add_log
+
+BASKETS_BASE_DIRECTORY = "baskets"
+CONFIG_FILE_DEFAULT_NAME = "config.json"
+
+if sys.platform in ("linux", "linux2", "darwin"):
+filesystem_location = '/etc/iotcollector'
+elif sys.platform == "win32":
+filesystem_location = os.path.join(os.path.dirname(__file__), os.pardir)
+
+
+def set_filesystem_location(new_location):
+global filesystem_location
+filesystem_location = new_location
+
+
+def init_file_system():
+try:
+if not os.path.exists(filesystem_location):
+os.makedirs(filesystem_location)
+except (Exception, OSError) as ex:
+print >> sys.stdout, ex
+add_log(50, ex)
+sys.exit(1)
diff --git a/clients/iotapi/src/Settings/iotlogger.py 
b/clients/iotapi/src/Settings/iotlogger.py
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -0,0 +1,38 @@
+import logging
+import sys
+
+import os
+
+if sys.platform in ("linux", "linux2", "darwin"):
+logging_location = '/var/log/iot/iotapi.log'
+elif sys.platform == "win32":
+logging_location = os.path.join(os.path.dirname(__file__), os.pardir, 
'iotapi.log')
+
+logger = logging.getLogger("IOTAPILog")
+
+
+def set_logging_location(new_location):
+global logging_location
+logging_location = new_location
+
+
+def init_logging():
+global logger
+try:
+logger = logging.getLogger("IOTLog")
+logger.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s 
- %(message)s')
+
+logging_path = os.path.dirname(logging_location)
+if not os.path.exists(logging_path):
+os.makedirs(logging_path)
+log_handler = logging.FileHandler(logging_location, mode='a+')
+log_handler.setFormatter(formatter)
+logger.addHandler(log_handler)
+except (Exception, OSError) as ex:
+print >> sys.stdout, ex
+sys.exit(1)
+
+
+def add_log(lvl, message, *args, **kwargs):
+logger.log(lvl, message, *args, **kwargs)
diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -0,0 +1,50 @@
+import getpass
+import sys
+
+import pymonetdb
+from Settings.iotlogger import add_log
+
+Connection = None
+
+
+def init_monetdb_connection(hostname, port, user_name, database):
+global Connection
+
+user_password = getpass.getpass(prompt='Insert password for user ' + 
user_name + ':')
+
+if user_password == '':
+user_password = 'monetdb'
+
+try:  # the autocommit is set to true so each statement will be independent
+Connection = pymonetdb.connect(hostname=hostname, port=port, 
username=user_name, password=user_password,
+   database=database, autocommit=True)
+Connection.execute("SET SCHEMA iot;")
+log_message = 'User %s connected successfully to database %s' % 
(user_name, database)
+print >> sys.stdout, log_message
+

MonetDB: iot - Corrected binary representations

[Pedro Ferreira]

Changeset: e2d39b39fc91 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e2d39b39fc91
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
Branch: iot
Log Message:

Corrected binary representations


diffs (truncated from 334 to 300 lines):

diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -6,5 +6,5 @@ pytz==2016.4
 rfc3987==1.3.5
 Sphinx==1.4.1
 sphinx-rtd-theme==0.1.9
-strict-rfc3339==0.6
+strict-rfc3339==0.7
 tzlocal==1.2.2
diff --git a/clients/iotclient/src/Streams/datatypes.py 
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -9,14 +9,17 @@ import re
 from abc import ABCMeta, abstractmethod
 from dateutil import parser
 
-from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX
+from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
 
 # Later check the byte order 
https://docs.python.org/2/library/struct.html#byte-order-size-and-alignment
 # Also check the consequences of aligment on packing HUGEINTs!
 # The null constants might change from system to system due to different CPU's
-ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
+LITTLE_ENDIAN_ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
+BIG_ENDIAN_ALIGNMENT = '>'
+UUID_SIZE = 16
 
 NIL_STRING = "\200"
+NIL_UUID = "----"
 
 INT8_MIN = 0x80
 INT16_MIN = 0x8000
@@ -51,12 +54,12 @@ class StreamDataType(object):
 def is_nullable(self):  # check if the column is nullable or not
 return self._is_nullable
 
+@abstractmethod
 def get_nullable_constant(self):  # get the nullable constant if the 
column is nullable
 return None
 
-@abstractmethod
 def set_default_value(self, default_value):  # set the default value 
representation in the data type
-self._default_value = None
+self._default_value = default_value
 
 def get_default_value(self):  # get the default value representation in 
the data type
 return self._default_value
@@ -120,9 +123,6 @@ class TextType(StreamDataType):
 def get_nullable_constant(self):
 return NIL_STRING
 
-def set_default_value(self, default_value):
-self._default_value = default_value
-
 def add_json_schema_entry(self, schema):
 super(TextType, self).add_json_schema_entry(schema)
 schema[self._column_name]['type'] = 'string'
@@ -137,28 +137,7 @@ class TextType(StreamDataType):
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
 string_pack = "".join(extracted_values)
-return struct.pack(ALIGNMENT + str(parameters['lengths_sum']) + 's', 
string_pack)
-
-
-class UUIDType(TextType):
-"""Covers: UUID"""
-
-def __init__(self, **kwargs):
-super(UUIDType, self).__init__(**kwargs)
-
-def add_json_schema_entry(self, schema):
-super(UUIDType, self).add_json_schema_entry(schema)
-schema[self._column_name]['pattern'] = UUID_REGEX
-
-def prepare_parameters(self):
-return {}
-
-def process_next_value(self, entry, counter, parameters, errors):
-return str(entry) + '\n'
-
-def pack_parsed_values(self, extracted_values, counter, parameters):
-string_pack = "".join(extracted_values)
-return struct.pack(ALIGNMENT + str(37 * counter) + 's', string_pack)
+return struct.pack(LITTLE_ENDIAN_ALIGNMENT + 
str(parameters['lengths_sum']) + 's', string_pack)
 
 
 class MACType(TextType):
@@ -171,15 +150,12 @@ class MACType(TextType):
 super(MACType, self).add_json_schema_entry(schema)
 schema[self._column_name]['pattern'] = MAC_ADDRESS_REGEX
 
-def prepare_parameters(self):
-return {}
-
 def process_next_value(self, entry, counter, parameters, errors):
 return str(entry) + '\n'
 
 def pack_parsed_values(self, extracted_values, counter, parameters):
 string_pack = "".join(extracted_values)
-return struct.pack(ALIGNMENT + str(18 * counter) + 's', string_pack)
+return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(18 * counter) + 's', 
string_pack)
 
 def process_sql_parameters(self, array):
 array[2] = 'char(17)'  # A MAC Address has 17 characters
@@ -196,32 +172,6 @@ class URLType(TextType):
 schema[self._column_name]['format'] = 'uri'
 
 
-class INet(TextType):
-"""Covers: Inet"""
-
-def __init__(self, **kwargs):
-super(INet, self).__init__(**kwargs)
-
-def add_json_schema_entry(self, schema):
-super(INet, self).add_json_schema_entry(schema)
-schema[self._column_name]['format'] = 'ipv4'
-
-
-class INetSix(TextType):
-"""Covers: Inet6"""
-
-def __init__(self, **kwargs):
- 

MonetDB: iot - Corrected basket searching

[Pedro Ferreira]

Changeset: b53bfefff4b3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b53bfefff4b3
Modified Files:
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Corrected basket searching


diffs (243 lines):

diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -5,15 +5,17 @@ from Utilities.customthreading import Pe
 
 from streamscontext import Streams_context, DataCellStream
 
-SWITCHER = [{'types': ['clob', 'char', 'varchar'], 'class': 'TextType'},
-{'types': ['boolean'], 'class': 'BooleanType'},
+SWITCHER = [{'types': ['clob', 'char', 'varchar', 'url'], 'class': 'TextType'},
 {'types': ['tinyint', 'smallint', 'int', 'bigint'], 'class': 
'SmallIntegerType'},
 {'types': ['hugeint'], 'class': 'HugeIntegerType'},
 {'types': ['real', 'double'], 'class': 'FloatType'},
 {'types': ['decimal'], 'class': 'DecimalType'},
+{'types': ['boolean'], 'class': 'BooleanType'},
 {'types': ['date'], 'class': 'DateType'},
 {'types': ['time'], 'class': 'TimeType'},
-{'types': ['timestamp'], 'class': 'TimestampType'}]
+{'types': ['timestamp'], 'class': 'TimestampType'},
+{'types': ['inet'], 'class': 'INetType'},
+{'types': ['uuid'], 'class': 'UUIDType'}]
 
 
 def init_stream_polling_thread(interval):
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -2,12 +2,14 @@ import struct
 
 import os
 from Settings.filesystem import get_baskets_base_location
+from Utilities.readwritelock import RWLock
 from WebSockets.websockets import notify_clients
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
 BASKETS_COUNT_FILE = 'count'
 
+
 def represents_int(s):
 try:
 int(s)
@@ -26,56 +28,95 @@ class StreamBasketsHandler(FileSystemEve
 def on_created(self, event):  # whenever a basket directory is created, 
notify to subscribed clients
 if isinstance(event, 'DirCreatedEvent'):
 basket_string = os.path.basename(os.path.normpath(event.src_path))
-self._stream.baskets.append_basket(basket_string)
-notify_clients(self._stream.schema_name, self._stream.stream_name)
+self._stream.append_basket(basket_string)
+notify_clients(self._stream.get_schema_name(), 
self._stream.get_stream_name())
 
+def on_deleted(self, event):
+if isinstance(event, 'DirDeletedEvent'):
+basket_string = os.path.basename(os.path.normpath(event.src_path))
+self._stream.delete_basket(basket_string)
 
-class DataCellStream(object):
+
+class IOTStream(object):
 """Representation of a stream"""
 
 def __init__(self, schema_name, stream_name, columns):
-self.schema_name = schema_name  # name of the schema
-self.stream_name = stream_name  # name of the stream
+self._schema_name = schema_name  # name of the schema
+self._stream_name = stream_name  # name of the stream
 self._columns = columns  # dictionary of name -> data_types
 self._base_path = os.path.join(get_baskets_base_location(), 
schema_name, stream_name)
-self.baskets = {}  # dictionary of basket_number -> total_tuples
+self._baskets = {}  # dictionary of basket_number -> total_tuples
 for name in os.listdir(self._base_path):
 self.append_basket(name)
+self._lock = RWLock()
 self._observer = Observer()
 self._observer.schedule(StreamBasketsHandler(stream=self), 
self._base_path, recursive=False)
 self._observer.start()
 
+def get_schema_name(self):
+return self._schema_name
+
+def get_stream_name(self):
+return self._stream_name
+
 def append_basket(self, path):
 if represents_int(path):
 with open(os.path.join(self._base_path, path)) as f:
 count = struct.unpack('i', f.read(4))[0]
-self.baskets[int(path)] = count
+self._lock.acquire_write()
+self._baskets[int(path)] = count
+self._lock.release()
 
-# TODO add delete basket
+def delete_basket(self, path):
+if represents_int(path):
+number = int(path)
+self._lock.acquire_write()
+if number in self._baskets:
+del self._baskets[number]
+self._lock.release()
+
+def get_next_basket_number_tuple(self, basket_number):

MonetDB: iot - Added offset/limit search

[Pedro Ferreira]

Changeset: 5b87e07b8c7f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b87e07b8c7f
Modified Files:
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/WebSockets/jsonschemas.py
Branch: iot
Log Message:

Added offset/limit search


diffs (136 lines):

diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -1,11 +1,12 @@
+import struct
+
 import os
 from Settings.filesystem import get_baskets_base_location
 from WebSockets.websockets import notify_clients
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
-from src.Streams.streamscontext import IOTStreams
-
+BASKETS_COUNT_FILE = 'count'
 
 def represents_int(s):
 try:
@@ -22,10 +23,10 @@ class StreamBasketsHandler(FileSystemEve
 super(StreamBasketsHandler, self).__init__()
 self._stream = stream
 
-def on_created(self, event):  # whenever a basket directory is created, 
notify
+def on_created(self, event):  # whenever a basket directory is created, 
notify to subscribed clients
 if isinstance(event, 'DirCreatedEvent'):
-basket_number = 
int(os.path.basename(os.path.normpath(event.src_path)))
-self._stream.baskets.append(basket_number)
+basket_string = os.path.basename(os.path.normpath(event.src_path))
+self._stream.baskets.append_basket(basket_string)
 notify_clients(self._stream.schema_name, self._stream.stream_name)
 
 
@@ -37,25 +38,59 @@ class DataCellStream(object):
 self.stream_name = stream_name  # name of the stream
 self._columns = columns  # dictionary of name -> data_types
 self._base_path = os.path.join(get_baskets_base_location(), 
schema_name, stream_name)
-self.baskets = [int(name) for name in os.listdir(self._base_path) if 
represents_int(name)]
+self.baskets = {}  # dictionary of basket_number -> total_tuples
+for name in os.listdir(self._base_path):
+self.append_basket(name)
 self._observer = Observer()
 self._observer.schedule(StreamBasketsHandler(stream=self), 
self._base_path, recursive=False)
 self._observer.start()
 
+def append_basket(self, path):
+if represents_int(path):
+with open(os.path.join(self._base_path, path)) as f:
+count = struct.unpack('i', f.read(4))[0]
+self.baskets[int(path)] = count
+
+# TODO add delete basket
+
 def read_tuples(self, basket_number, limit, offset):
-if basket_number not in self.baskets:
-concatenated_name = 
IOTStreams.get_context_entry_name(self.schema_name, self.stream_name)
-raise Exception('Stream ' + concatenated_name + ' does not contain 
basket ' + str(basket_number))
+results = {column: [] for column in self._columns.keys()}
+current_basket = int(basket_number)
+read_tuples = 0
+finished = False
 
-results = []
-base_read_path = os.path.join(self._base_path, str(basket_number))
-for key, column in self._columns.iteritems():
-next_file_name = os.path.join(base_read_path, key)
-open_string = 'r'
-if not column.is_file_mode_binary():
-open_string += 'u'
-file_pointer = open(next_file_name, open_string)
-results.append(column.read_next_batch(file_pointer, 100))
+while True:
+if current_basket not in self.baskets:
+finished = True
+break
+offset -= self.baskets[current_basket]
+if offset < 0:
+break
+current_basket += 1
 
-packed_results = zip(*results)  # TODO check if this is viable, it 
could be 1000 tuples
-# notify_clients(self._schema_name, self._stream_name, packed_results)
+if not finished:
+offset = abs(offset)
+
+while True:
+if current_basket not in self.baskets:
+break
+next_path = os.path.join(self._base_path, str(current_basket))
+next_read_size = min(self.baskets[current_basket], limit)
+
+for key, column in self._columns.iteritems():
+next_file_name = os.path.join(next_path, key)
+open_string = 'r'
+if not column.is_file_mode_binary():
+open_string += 'u'
+file_pointer = open(next_file_name, open_string)
+results[key].append(column.read_next_batch(file_pointer, 
offset, next_read_size))
+
+offset = 0
+current_basket += 1
+read_tuples += next_read_size
+limit -= self.baskets[current_basket]
+if limit <= 0:
+

MonetDB: iot - Introducing publish/subscribe pattern for streams.

[Pedro Ferreira]

Changeset: 409630148656 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=409630148656
Added Files:
clients/iotapi/src/Utilities/readwritelock.py
clients/iotapi/src/WebSockets/jsonschemas.py
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/WebSockets/websockets.py
Branch: iot
Log Message:

Introducing publish/subscribe pattern for streams.


diffs (truncated from 334 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -1,4 +1,5 @@
 git+https://github.com/dpallot/simple-websocket-server.git
+jsonschema==2.5.1
 pymonetdb==0.1.1
 python-dateutil==2.5.3
 pytz==2016.4
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,7 +1,7 @@
 import struct
-from abc import ABCMeta, abstractmethod
 from datetime import date, time, datetime
 
+from abc import ABCMeta, abstractmethod
 from dateutil.relativedelta import relativedelta
 
 ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
@@ -35,9 +35,6 @@ class StreamDataType(object):
 def read_next_batch(self, file_pointer, count):
 return []
 
-def fetch_new_tuples(self, count):
-file_pointer = open(self._location, 'rb')
-
 
 class TextType(StreamDataType):
 """Covers: CHAR, VARCHAR, CLOB"""
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -23,9 +23,13 @@ def init_stream_polling_thread(interval)
 
 # elem[0] is schema. elem[1] is name, elem[2] is column name, elem[3] is type, 
elem[4] is location, elem[5] is typewidth
 def stream_polling():
+current_streams = Streams_context.get_existing_streams()
+retained_streams = []
+new_streams = {}
 array = fetch_streams()  # TODO check whenever stream's columns are updated
-for key, group in groupby(array, lambda x: x[0] + '.' + x[1]):
-if not Streams_context.is_stream_in_context(key):
+
+for key, group in groupby(array, lambda x: 
Streams_context.get_context_entry_name(x[0], x[1])):
+if key not in current_streams:
 columns = {}
 
 for elem in group:
@@ -33,5 +37,9 @@ def stream_polling():
 kwargs = {'name': elem[2], 'type': elem[3], 'location': 
elem[4], 'typewidth': elem[5]}
 new_column = reflection_class(kwargs)
 columns[elem[2]] = new_column
+new_streams[key] = DataCellStream(key, columns)
+else:
+retained_streams.append(key)
 
-Streams_context.add_stream(key, DataCellStream(key, columns))
+retained_streams_final = [key for key in current_streams if key in 
retained_streams]
+Streams_context.merge_context(retained_streams_final, new_streams)
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -1,36 +1,61 @@
 import os
-
 from Settings.filesystem import get_baskets_base_location
+from WebSockets.websockets import notify_clients
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
+from src.Streams.streamscontext import IOTStreams
+
+
+def represents_int(s):
+try:
+int(s)
+return True
+except ValueError:
+return False
+
 
 class StreamBasketsHandler(FileSystemEventHandler):
+"""File watcher of a stream"""
+
 def __init__(self, stream):
 super(StreamBasketsHandler, self).__init__()
 self._stream = stream
 
 def on_created(self, event):  # whenever a basket directory is created, 
notify
 if isinstance(event, 'DirCreatedEvent'):
-self._stream.read_new_tuples(event.src_path)
+basket_number = 
int(os.path.basename(os.path.normpath(event.src_path)))
+self._stream.baskets.append(basket_number)
+notify_clients(self._stream.schema_name, self._stream.stream_name)
 
 
 class DataCellStream(object):
 """Representation of a stream"""
 
 def __init__(self, schema_name, stream_name, columns):
-self._schema_name = schema_name  # name of the schema
-self._stream_name = stream_name  # name of the stream
+self.schema_name = schema_name  # name of the schema
+self.stream_name = stream_name  # name of the stream
 self._columns = columns  # dictionary of name -> data_types
 self._base_path = os.path.join(get_baskets_base_location(), 

MonetDB: iot - Added file watching

[Pedro Ferreira]

Changeset: 233ce5ea7032 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=233ce5ea7032
Added Files:
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/WebSockets/__init__.py
clients/iotapi/src/WebSockets/websockets.py
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Added file watching


diffs (truncated from 451 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -1,7 +1,8 @@
 git+https://github.com/dpallot/simple-websocket-server.git
+pymonetdb==0.1.1
 python-dateutil==2.5.3
 pytz==2016.4
-pymonetdb==0.1.1
-tzlocal==1.2.2
 Sphinx==1.4.1
 sphinx-rtd-theme==0.1.9
+tzlocal==1.2.2
+watchdog==0.8.3
diff --git a/clients/iotapi/src/Settings/filesystem.py 
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -5,24 +5,35 @@ import os
 from iotlogger import add_log
 
 BASKETS_BASE_DIRECTORY = "baskets"
-CONFIG_FILE_DEFAULT_NAME = "config.json"
 
 if sys.platform in ("linux", "linux2", "darwin"):
-filesystem_location = '/etc/iotcollector'
+Filesystem_Location = '/etc/iotcollector'
 elif sys.platform == "win32":
-filesystem_location = os.path.join(os.path.dirname(__file__), os.pardir)
+Filesystem_Location = os.path.join(os.path.dirname(__file__), os.pardir)
+
+Baskets_Location = None
 
 
 def set_filesystem_location(new_location):
-global filesystem_location
-filesystem_location = new_location
+global Filesystem_Location
+Filesystem_Location = new_location
 
 
 def init_file_system():
+global Baskets_Location
+
 try:
-if not os.path.exists(filesystem_location):
-os.makedirs(filesystem_location)
+Baskets_Location = os.path.join(Filesystem_Location, 
BASKETS_BASE_DIRECTORY)
+if not os.path.exists(Baskets_Location):
+os.makedirs(Baskets_Location)
+
+if not os.path.exists(Filesystem_Location):
+os.makedirs(Filesystem_Location)
 except (Exception, OSError) as ex:
 print >> sys.stdout, ex
 add_log(50, ex)
 sys.exit(1)
+
+
+def get_baskets_base_location():
+return Baskets_Location
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,4 +1,3 @@
-import itertools
 import struct
 from abc import ABCMeta, abstractmethod
 from datetime import date, time, datetime
@@ -27,25 +26,31 @@ class StreamDataType(object):
 def __init__(self, **kwargs):
 self._column_name = kwargs['name']  # name of the column
 self._data_type = kwargs['type']  # SQL name of the type
-self._location = kwargs['location'] + '.tail'  # Location of the file
+# self._location = kwargs['location'] + '.tail'  # Location of the file
+
+def is_file_mode_binary(self):
+return True
 
 @abstractmethod
 def read_next_batch(self, file_pointer, count):
 return []
 
-def to_json_representation(self):
-return {'name': self._column_name, 'type': self._data_type}
+def fetch_new_tuples(self, count):
+file_pointer = open(self._location, 'rb')
 
 
 class TextType(StreamDataType):
-"""Covers: TEXT, STRING, CLOB and CHARACTER LARGE OBJECT"""
+"""Covers: CHAR, VARCHAR, CLOB"""
 
 def __init__(self, **kwargs):
 super(TextType, self).__init__(**kwargs)
 self._nullable_constant = NIL_STRING
 
+def is_file_mode_binary(self):
+return False
+
 def read_next_batch(self, file_pointer, count):
-array = list(itertools.islice(file_pointer, count))
+array = file_pointer.readlines()
 return map(lambda x: None if x == self._nullable_constant else x[:-1], 
array)
 
 
@@ -62,7 +67,7 @@ class BooleanType(StreamDataType):
 
 
 class SmallIntegerType(StreamDataType):
-"""Covers: TINYINT, SMALLINT, INT[EGER], BIGINT"""
+"""Covers: TINYINT, SMALLINT, INTEGER, BIGINT"""
 
 def __init__(self, **kwargs):
 super(SmallIntegerType, self).__init__(**kwargs)
@@ -98,7 +103,7 @@ class HugeIntegerType(StreamDataType):
 
 
 class FloatType(StreamDataType):
-"""Covers: REAL, FLOAT and DOUBLE"""
+"""Covers: REAL, DOUBLE"""
 
 def __init__(self, **kwargs):

MonetDB: iot - Finished data reconstruction

[Pedro Ferreira]

Changeset: a91cdecbf145 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a91cdecbf145
Modified Files:
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streams.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Finished data reconstruction


diffs (truncated from 334 to 300 lines):

diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,12 +1,13 @@
 import struct
+from abc import ABCMeta, abstractmethod
 from datetime import date, time, datetime
 
-from abc import ABCMeta, abstractmethod
 from dateutil.relativedelta import relativedelta
 
-ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
+LITTLE_ENDIAN_ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
 
 NIL_STRING = "\200\n"  # added newline for performance
+NIL_UUID = "----"
 
 INT8_MIN = 0x80
 INT16_MIN = 0x8000
@@ -20,24 +21,40 @@ DOUBLE_NAN = struct.unpack('d', '\xff\xf
 
 
 class StreamDataType(object):
-"""MonetDB's data types for validation base class"""
+"""MonetDB's data types for reading base class"""
 __metaclass__ = ABCMeta
 
 def __init__(self, **kwargs):
 self._column_name = kwargs['name']  # name of the column
 self._data_type = kwargs['type']  # SQL name of the type
-# self._location = kwargs['location'] + '.tail'  # Location of the file
 
 def is_file_mode_binary(self):
 return True
 
 @abstractmethod
-def read_next_batch(self, file_pointer, count):
-return []
+def skip_tuples(self, file_pointer, offset):
+pass
+
+@abstractmethod
+def read_next_batch(self, file_pointer, limit):
+pass
+
+def read_next_tuples(self, file_name, offset, read_size):
+open_string = 'r'
+if not self.is_file_mode_binary():
+open_string += 'u'
+file_pointer = open(file_name, open_string)
+
+if offset > 0:
+self.skip_tuples(file_pointer, offset)
+
+results = self.read_next_batch(file_pointer, read_size)
+file_pointer.close()
+return results
 
 
 class TextType(StreamDataType):
-"""Covers: CHAR, VARCHAR, CLOB"""
+"""Covers: CHAR, VARCHAR, CLOB and URL"""
 
 def __init__(self, **kwargs):
 super(TextType, self).__init__(**kwargs)
@@ -46,9 +63,69 @@ class TextType(StreamDataType):
 def is_file_mode_binary(self):
 return False
 
-def read_next_batch(self, file_pointer, count):
-array = file_pointer.readlines()
-return map(lambda x: None if x == self._nullable_constant else x[:-1], 
array)
+def skip_tuples(self, file_pointer, offset):
+for _ in xrange(offset):
+next(file_pointer)
+
+def read_next_batch(self, file_pointer, limit):
+array = []
+for _ in xrange(limit):
+next_line = next(file_pointer)
+if next_line == self._nullable_constant:
+array.append(None)
+else:
+array.append(next_line[:-1])  # remove newline
+return array
+
+
+class INetType(StreamDataType):
+"""Covers: Inet"""
+
+def __init__(self, **kwargs):
+super(INetType, self).__init__(**kwargs)
+
+def skip_tuples(self, file_pointer, offset):
+file_pointer.seek(offset << 3)
+
+def read_next_batch(self, file_pointer, limit):
+results = []
+read_size = limit << 3
+array = struct.unpack(LITTLE_ENDIAN_ALIGNMENT + str(read_size) + 'B', 
file_pointer.read(read_size))
+iterator = iter(array)
+
+for _ in xrange(limit):
+next_ipv4 = [next(iterator) for _ in xrange(8)]
+if next_ipv4[7] == 1:  # check nil value
+results.append(None)
+else:
+parsed_ip = '.'.join([str(next_ipv4[0]), str(next_ipv4[1]), 
str(next_ipv4[2]), str(next_ipv4[3])])
+results.append(parsed_ip + '/' + str(next_ipv4[4]))
+return results
+
+
+class UUIDType(StreamDataType):
+"""Covers: UUID"""
+
+def __init__(self, **kwargs):
+super(UUIDType, self).__init__(**kwargs)
+
+def skip_tuples(self, file_pointer, offset):
+file_pointer.seek(offset << 4)
+
+def read_next_batch(self, file_pointer, limit):
+results = []
+read_size = limit << 4
+array = struct.unpack(LITTLE_ENDIAN_ALIGNMENT + str(read_size) + 'B', 
file_pointer.read(read_size))
+iterator = iter(array)
+
+for _ in xrange(limit):
+next_uuid = ''.join(map(lambda x: "%02x" % x, [next(iterator) for 
_ in xrange(16)]))
+next_uuid = ''.join([next_uuid[:8], '-', next_uuid[8:12], '-', 
next_uuid[12:16], '-', next_uuid[16:20],
+ '-', next_uuid[20:]])
+if next_uuid == NIL_UUID:
+ 

MonetDB: iot - Finished stream reading

[Pedro Ferreira]

Changeset: 5b90b8e6494a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b90b8e6494a
Modified Files:
clients/iotapi/src/Settings/mapiconnection.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/WebSockets/jsonschemas.py
clients/iotapi/src/WebSockets/websockets.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Finished stream reading


diffs (truncated from 661 to 300 lines):

diff --git a/clients/iotapi/src/Settings/mapiconnection.py 
b/clients/iotapi/src/Settings/mapiconnection.py
--- a/clients/iotapi/src/Settings/mapiconnection.py
+++ b/clients/iotapi/src/Settings/mapiconnection.py
@@ -47,4 +47,3 @@ def fetch_streams():
 except BaseException as ex:
 print >> sys.stdout, ex
 add_log(50, ex)
-sys.exit(1)
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -28,8 +28,8 @@ class StreamBasketsHandler(FileSystemEve
 def on_created(self, event):  # whenever a basket directory is created, 
notify to subscribed clients
 if isinstance(event, 'DirCreatedEvent'):
 basket_string = os.path.basename(os.path.normpath(event.src_path))
-self._stream.append_basket(basket_string)
-notify_clients(self._stream.get_schema_name(), 
self._stream.get_stream_name())
+count = self._stream.append_basket(basket_string)
+notify_clients(self._stream.get_schema_name(), 
self._stream.get_stream_name(), count)
 
 def on_deleted(self, event):
 if isinstance(event, 'DirDeletedEvent'):
@@ -49,6 +49,7 @@ class IOTStream(object):
 for name in os.listdir(self._base_path):
 self.append_basket(name)
 self._lock = RWLock()
+
 self._observer = Observer()
 self._observer.schedule(StreamBasketsHandler(stream=self), 
self._base_path, recursive=False)
 self._observer.start()
@@ -66,6 +67,8 @@ class IOTStream(object):
 self._lock.acquire_write()
 self._baskets[int(path)] = count
 self._lock.release()
+return count
+return 0
 
 def delete_basket(self, path):
 if represents_int(path):
@@ -126,7 +129,6 @@ class IOTStream(object):
 offset = 0
 current_basket_number += 1
 
-# TODO check if this is viable, it could be 1000 tuples
-keys = results.keys()
+keys = results.keys()  # TODO check if this is viable for many tuples!!
 tuples = [dict(zip(keys, values)) for values in zip(*(results[k] for k 
in keys))]
 return {'total': read_tuples, 'tuples': tuples}
diff --git a/clients/iotapi/src/Streams/streamscontext.py 
b/clients/iotapi/src/Streams/streamscontext.py
--- a/clients/iotapi/src/Streams/streamscontext.py
+++ b/clients/iotapi/src/Streams/streamscontext.py
@@ -5,7 +5,7 @@ from WebSockets.websockets import desubs
 
 
 class IOTStreams(object):
-"""Stream's context"""
+"""Streams context"""
 
 @classmethod
 def get_context_entry_name(cls, schema_name, stream_name):
diff --git a/clients/iotapi/src/WebSockets/jsonschemas.py 
b/clients/iotapi/src/WebSockets/jsonschemas.py
--- a/clients/iotapi/src/WebSockets/jsonschemas.py
+++ b/clients/iotapi/src/WebSockets/jsonschemas.py
@@ -1,6 +1,11 @@
-PUBSUB_STREAMS_SCHEMA = {
+SUBSCRIBE_OPTS = ["sub", "subscribe"]
+UNSUBSCRIBE_OPTS = ["unsub", "unsubscribe"]
+CONCAT_SUB_OPTS = SUBSCRIBE_OPTS + UNSUBSCRIBE_OPTS
+READ_OPTS = ["read"]
+
+CLIENTS_INPUTS_SCHEMA = {
 "title": "JSON schema publish/subscribe streams",
-"description": "Validate data inserted",
+"description": "Validate clients inputs",
 "$schema": "http://json-schema.org/draft-04/schema#;,
 "type": "object",
 
@@ -8,7 +13,7 @@ PUBSUB_STREAMS_SCHEMA = {
 "properties": {
 "schema": {"type": "string"},
 "stream": {"type": "string"},
-"action": {"type": "string", "enum": ["sub", "subscribe", "desub", 
"desubscribe"]},
+"action": {"type": "string", "enum": CONCAT_SUB_OPTS},
 },
 "required": ["schema", "stream", "action"],
 "additionalProperties": False
@@ -22,7 +27,7 @@ PUBSUB_STREAMS_SCHEMA = {
 "properties": {
 "schema": {"type": "string"},
 "stream": {"type": "string"},
-"action": {"type": "string", "enum": ["read"]},
+"action": {"type": "string", "enum": READ_OPTS},
 "basket": {"type": "integer", "minimum": 1, "default": 1},
 

MonetDB: iot - Corrections on the documentation

[Pedro Ferreira]

Changeset: 153f10e5c2b6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=153f10e5c2b6
Modified Files:
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/src/main.py
Branch: iot
Log Message:

Corrections on the documentation


diffs (102 lines):

diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -30,7 +30,7 @@ Set the filesystem directory where the b
 
 **-l  - -log=**
 
-Location of logfile. On the logfile is reported when streams are created or 
removed, when tuples are inserted and when the baskets are flushed. By default 
in UNIX systems is :code:`/var/log/iot/iot.log`, while on Windows is the 
:code:`iot.log` on the directory where the :code:`main.py` script was invoked.
+Location of logfile. On the logfile is reported when streams are created or 
removed, when tuples are inserted and when the baskets are flushed. By default 
in UNIX systems is :code:`/var/log/iot/iot.log`, while on Windows is the 
:code:`iot.log` on the directory where the :code:`main.py` script was called.
 
 **-c  - -config=**
 
@@ -41,18 +41,18 @@ Host Identifier
 
 If the *useidentifier* parameter is provided, an extra column on streams will 
be added with a custom name of the host for later identification. 
 
-**-ui  - -useidentifier=**
+**-ui  - -useidentifier**
 
-Boolean to indicate if the extra column will be added.
+Use a host identifier for every new stream.
 
 **-in  - -name=**
 
-Custom name of the host. By default is the host's MAC address.
+Host identifier name. By default is the host's MAC address.
 
 Web Server Listening
 
 
-For security purposes, two web servers are created by the application. On the 
administration server are present operations to create and delete streams, 
while on application server the insert operations are present instead. The 
administration server should be listening on the host only, while the 
application server should be listening to all interfaces.
+For security purposes, two web servers are created by the application. On the 
administration server it's present operations to create and delete streams, 
while on application server the insert operations are present instead. The 
administration server should be listening on the host only, while the 
application server should be listening to all interfaces.
 
 **-ih  - -ihost=**
 
@@ -64,18 +64,18 @@ Listening port of the application (IOT) 
 
 **-ah  - -ahost=**
 
-Listening host of the administration (IOT) server. By default is on 
:code:`127.0.0.1`.
+Listening host of the administration server. By default is on 
:code:`127.0.0.1`.
 
 **-ap  - -aport=**
 
-Listening port of the administration (IOT) server. By default is on port 
:code:`8001`.
+Listening port of the administration server. By default is on port 
:code:`8001`.
 
 Database Connection
 ---
 
 Credentials for the MAPI connection to MonetDB database.
 
-.. note:: The user password will be prompted during the initialization of the 
server.
+.. note:: The user's password will be prompted during the initialization of 
the server.
 
 **-d  - -host=**
 
diff --git a/clients/iotclient/documentation/streams_data_types.rst 
b/clients/iotclient/documentation/streams_data_types.rst
--- a/clients/iotclient/documentation/streams_data_types.rst
+++ b/clients/iotclient/documentation/streams_data_types.rst
@@ -38,7 +38,7 @@ A *Media Access Control Address* identif
 URL
 ---
 
-An *Uniform Resource Locator* as a specific type of an URI is validated 
according to `RFC 3987 `_. The insertion 
must also be a JSON String.
+An *Uniform Resource Locator* as a specific type of an URI is validated 
according to `RFC 3987 `_. The insertion 
must be a JSON String.
 
 Inet
 
@@ -117,9 +117,9 @@ A regular date in the Gregorian Calendar
 Time
 
 
-The time of day expressed in format :code:`HH:MM:SS.sss`. If the pair 
:code:`"timezone": false` is provided, the timezone will be truncated.
+The time of day expressed in format :code:`HH:MM:SS.sss`. If the pair 
:code:`"timezone": false` is provided, the timezone will be discarded.
 
 Timestamp
 -
 
-A timestamp according to `RFC 3339 `_ 
with timezone. The regular expression is the standard `ISO 8601 with timezone 
format `_. If the pair 
:code:`"timezone": false` is provided, the timezone will be truncated.
+A timestamp according to `RFC 3339 `_ 
with timezone. The regular expression is the standard `ISO 8601 with timezone 
format `_. If the pair 

MonetDB: iot - Added option to add hostname as a stream column.

[Pedro Ferreira]

Changeset: fd01b17e9fea for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fd01b17e9fea
Modified Files:
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Added option to add hostname as a stream column.


diffs (truncated from 306 to 300 lines):

diff --git a/clients/iotclient/src/Flask/app.py 
b/clients/iotclient/src/Flask/app.py
--- a/clients/iotclient/src/Flask/app.py
+++ b/clients/iotclient/src/Flask/app.py
@@ -1,7 +1,7 @@
 from flask import Flask
 from flask_restful import Api
 
-from restresources import StreamInput, StreamsInfo, StreamsHandling  # , 
ServerHandler
+from restresources import StreamInput, StreamsInfo, StreamsHandling
 
 
 def start_flask_iot_app(host, port):
@@ -21,5 +21,4 @@ def start_flask_admin_app(host, port):
 
 admin_api.add_resource(StreamsInfo, '/streams')
 admin_api.add_resource(StreamsHandling, '/context')
-#  admin_api.add_resource(ServerHandler, '/server')
 admin_app.run(host=host, port=port, threaded=True)
diff --git a/clients/iotclient/src/Settings/filesystem.py 
b/clients/iotclient/src/Settings/filesystem.py
--- a/clients/iotclient/src/Settings/filesystem.py
+++ b/clients/iotclient/src/Settings/filesystem.py
@@ -5,22 +5,24 @@ from Utilities.filecreator import create
 
 BASKETS_BASE_DIRECTORY = "baskets"
 CONFIG_FILE_DEFAULT_NAME = "config.json"
-Baskets_Base_Location = None
-Config_File_Location = None
 
 if sys.platform in ("linux", "linux2", "darwin"):
 filesystem_location = '/etc/iotcollector'
 elif sys.platform == "win32":
 filesystem_location = os.path.dirname(os.path.realpath(__file__))
 
+Baskets_Base_Location = None
+Config_File_Location = None
+Host_Identifier = None
+
 
 def set_filesystem_location(new_location):
 global filesystem_location
 filesystem_location = new_location
 
 
-def init_file_system(new_configfile_location=None):
-global Baskets_Base_Location, Config_File_Location
+def init_file_system(host_identifier=None, new_configfile_location=None):
+global Baskets_Base_Location, Config_File_Location, Host_Identifier
 
 try:
 Baskets_Base_Location = os.path.join(filesystem_location, 
BASKETS_BASE_DIRECTORY)
@@ -32,6 +34,8 @@ def init_file_system(new_configfile_loca
 else:
 Config_File_Location = create_file_if_not_exists(
 os.path.join(filesystem_location, CONFIG_FILE_DEFAULT_NAME), 
hidden=False, init_text='[]')
+
+Host_Identifier = host_identifier
 except (Exception, OSError) as ex:
 print >> sys.stderr, ex
 sys.exit(1)
@@ -43,3 +47,7 @@ def get_baskets_base_location():
 
 def get_configfile_location():
 return Config_File_Location
+
+
+def get_host_identifier():
+return Host_Identifier
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -28,11 +28,15 @@ def mapi_create_stream(schema, stream, c
 Connection.execute("CREATE SCHEMA " + schema + ";")
 except:
 pass
-sql_str = ''.join(["CREATE STREAM TABLE ", schema, ".", stream, " (", 
columns, ");"])
-Connection.execute(sql_str)
+
+try:  # attempt to create te stream table
+Connection.execute(''.join(["CREATE STREAM TABLE ", schema, ".", 
stream, " (", columns, ");"]))
+except:
+pass
 
 
 def mapi_flush_baskets(schema, stream, baskets):
-# this procedure does not work yet. Have to check it with Martin
-# Connection.execute(''.join(["CALL iot.push(\"", schema, "\",\"", stream, 
"\",\"", baskets, "\");"]))
-pass
+try:
+Connection.execute(''.join(["CALL iot.push(\"", schema, "\",\"", 
stream, "\",\"", baskets, "\");"]))
+except:
+pass
diff --git a/clients/iotclient/src/Streams/streams.py 
b/clients/iotclient/src/Streams/streams.py
--- a/clients/iotclient/src/Streams/streams.py
+++ b/clients/iotclient/src/Streams/streams.py
@@ -1,16 +1,14 @@
 import os
 from collections import defaultdict
 
-from datatypes import TimestampType, DataValidationException
+from datatypes import TimestampType, TextType, DataValidationException
 from flushing import TimeBasedFlushing, TupleBasedFlushing
 from Settings.mapiconnection import mapi_create_stream, mapi_flush_baskets
-from Settings.filesystem import get_baskets_base_location
+from Settings.filesystem import get_baskets_base_location, get_host_identifier
 from Settings.iotlogger import add_log
 from Utilities.filecreator import create_file_if_not_exists, 
get_hidden_file_name
 from Utilities.readwritelock import RWLock
 
-IMPLICIT_TIMESTAMP_COLUMN_NAME = 'implicit_timestamp'
-
 
 def represents_int(s):
 try:
@@ -19,6 +17,23 @@ def represents_int(s):
 

MonetDB: iot - Corrected several possible bugs, added decimal/nu...

[Pedro Ferreira]

Changeset: d8f1402029e1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d8f1402029e1
Removed Files:
clients/iotclient/src/Utilities/numberutilities.py
Modified Files:
clients/iotclient/README
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/semanticvalidation.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Corrected several possible bugs, added decimal/numeric serialization.


diffs (truncated from 439 to 300 lines):

diff --git a/clients/iotclient/README b/clients/iotclient/README
--- a/clients/iotclient/README
+++ b/clients/iotclient/README
@@ -23,8 +23,7 @@ REST resources available on iot webserve
 
 Currently supports most of MonetDB datatypes. An implicit Timestamp column is 
added with the tuples arrival date. The flushing of the baskets can be either 
Tuple or Time based, as specied on the streams' creation.
 
-More details coming soon.
+I will migrate this documentation into pydocs soon :)
 
 
 Maintainer: Pedro Ferreira at CWI, email: p.e.ferre...@cwi.nl
-
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -6,11 +6,11 @@ from flask import request
 from flask_restful import Resource
 from jsonschema import Draft4Validator, FormatChecker
 from tzlocal import get_localzone
-from src.Streams.jsonschemas import CREATE_STREAMS_SCHEMA, 
DELETE_STREAMS_SCHEMA
-from src.Streams.streamscontext import IOTStreamsException, IOTStreams
+from Streams.jsonschemas import CREATE_STREAMS_SCHEMA, DELETE_STREAMS_SCHEMA
+from Streams.streamscontext import IOTStreamsException, IOTStreams
 
 Stream_context = IOTStreams()
-local_tz = get_localzone()  # for the correction of dates we must add the 
timezone
+local_tz = get_localzone()  # for the correction of dates we must add the 
system's timezone
 
 
 class StreamInput(Resource):
diff --git a/clients/iotclient/src/Settings/filesystem.py 
b/clients/iotclient/src/Settings/filesystem.py
--- a/clients/iotclient/src/Settings/filesystem.py
+++ b/clients/iotclient/src/Settings/filesystem.py
@@ -2,6 +2,8 @@ import os
 import sys
 
 baskets_base_location = None
+BASKETS_BASE_DIRECTORY = "baskets"
+
 
 if sys.platform in ("linux", "linux2", "darwin"):
 filesystem_location = '/etc/iotcollector'
@@ -12,16 +14,28 @@ elif sys.platform == "win32":
 def set_filesystem_location(new_location):
 global filesystem_location
 
-if os.path.isdir(new_location):
-filesystem_location = new_location
-else:
-print >> sys.stderr, "The provided filesystem doesn't exist!"
+try:
+if os.path.isdir(new_location):
+filesystem_location = new_location
+else:
+print >> sys.stderr, "The provided filesystem doesn't exist!"
+sys.exit(1)
+except (Exception, OSError) as ex:
+print >> sys.stderr, ex
 sys.exit(1)
 
 
 def init_file_system():
 global baskets_base_location
 
-baskets_base_location = os.path.join(filesystem_location, "baskets")
-if not os.path.exists(baskets_base_location):
-os.makedirs(baskets_base_location)
+try:
+baskets_base_location = os.path.join(filesystem_location, 
BASKETS_BASE_DIRECTORY)
+if not os.path.exists(baskets_base_location):
+os.makedirs(baskets_base_location)
+except (Exception, OSError) as ex:
+print >> sys.stderr, ex
+sys.exit(1)
+
+
+def get_baskets_base_location():
+return baskets_base_location
diff --git a/clients/iotclient/src/Settings/iotlogger.py 
b/clients/iotclient/src/Settings/iotlogger.py
--- a/clients/iotclient/src/Settings/iotlogger.py
+++ b/clients/iotclient/src/Settings/iotlogger.py
@@ -29,6 +29,6 @@ def init_logging():
 log_handler = logging.FileHandler(logging_location, mode='a+')
 log_handler.setFormatter(formatter)
 logger.addHandler(log_handler)
-except Exception as ex:
-print >> sys.stderr, ex.message
+except (Exception, OSError) as ex:
+print >> sys.stderr, ex
 sys.exit(1)
diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -1,5 +1,6 @@
 import sys
 import pymonetdb
+import getpass
 
 Connection = None
 
@@ -7,11 

MonetDB: iot - Added rst files for documentation of iot client a...

[Pedro Ferreira]

Changeset: cdc459ebce61 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cdc459ebce61
Added Files:
clients/iotclient/documentation/conf.py
clients/iotclient/documentation/future_work.rst
clients/iotclient/documentation/index.rst
clients/iotclient/documentation/introduction.rst
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/documentation/streams_creation.rst
clients/iotclient/documentation/streams_data_types.rst
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Utilities/filecreator.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Added rst files for documentation of iot client and logging of errors


diffs (truncated from 918 to 300 lines):

diff --git a/clients/iotclient/documentation/conf.py 
b/clients/iotclient/documentation/conf.py
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/documentation/conf.py
@@ -0,0 +1,370 @@
+# -*- coding: utf-8 -*-
+#
+# MonetDB IOT WebServer documentation build configuration file, created by
+# sphinx-quickstart on Wed Apr 20 09:44:47 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration 
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+'sphinx.ext.autodoc',
+'sphinx.ext.doctest',
+'sphinx.ext.intersphinx',
+'sphinx.ext.todo',
+'sphinx.ext.coverage',
+'sphinx.ext.mathjax',
+'sphinx.ext.ifconfig',
+'sphinx.ext.viewcode',
+'sphinx.ext.githubpages',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'MonetDB IOT WebServer'
+copyright = u'2016, MonetDB Solutions'
+author = u'Pedro Ferreira'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = u'1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as 

MonetDB: iot - Merged with Martin code

[Pedro Ferreira]

Changeset: cac2e7880416 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cac2e7880416
Modified Files:
clients/iotclient/documentation/conf.py
sql/backends/monet5/iot/Tests/iot05.sql
sql/backends/monet5/iot/Tests/iot10.sql
sql/backends/monet5/iot/petrinet.c
sql/backends/monet5/iot/petrinet.h
Branch: iot
Log Message:

Merged with Martin code


diffs (204 lines):

diff --git a/clients/iotclient/documentation/conf.py 
b/clients/iotclient/documentation/conf.py
--- a/clients/iotclient/documentation/conf.py
+++ b/clients/iotclient/documentation/conf.py
@@ -276,7 +276,7 @@ man_pages = [
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
-#  dir menu entry, description, c  ategory)
+#  dir menu entry, description, category)
 texinfo_documents = [
 (master_doc, 'MonetDBIOTWebServer', u'MonetDB IOT WebServer Documentation',
  author, 'MonetDBIOTWebServer', 'One line description of project.',
diff --git a/sql/backends/monet5/iot/Tests/iot05.sql 
b/sql/backends/monet5/iot/Tests/iot05.sql
--- a/sql/backends/monet5/iot/Tests/iot05.sql
+++ b/sql/backends/monet5/iot/Tests/iot05.sql
@@ -13,6 +13,8 @@ set cnt = 0;
 -- this way their naming becomes easier, and mult-statement
 -- actions are better supported.
 
+--However, these queries won't run because the SQL context
+--holding the variables is not generally known
 create procedure clk1()
 begin
set hbclk1 = hbclk1+1;
@@ -21,14 +23,19 @@ end;
 create procedure clk3()
 begin
set hbclk1 = hbclk1+1;
-   set hbclk1 = hbclk1+1;
+   set hbclk2 = hbclk2+2;
--set cnt =(select count(*) from stmp);
 end;
 
 -- alternative is a simple query
 call iot.query('iot','clk1');
 call iot.query('iot','clk3');
-call iot.query('select 1;');
 
 select * from  iot.baskets();
 select * from  iot.queries();
+
+select hbclk1, hbclk2;
+call iot.activate();
+select hbclk1, hbclk2;
+
+
diff --git a/sql/backends/monet5/iot/Tests/iot10.sql 
b/sql/backends/monet5/iot/Tests/iot10.sql
--- a/sql/backends/monet5/iot/Tests/iot10.sql
+++ b/sql/backends/monet5/iot/Tests/iot10.sql
@@ -1,19 +1,24 @@
--- introduce a heartbeat query
+-- use accumulated aggregation
 set schema iot;
-set optimizer='iot_pipe';
 
 create table tmp_aggregate(tmp_total decimal(8,2), tmp_count decimal(8,2));
 insert into tmp_aggregate values(0.0,0.0);
 
-
 create procedure collector()
 begin
update tmp_aggregate
-   set tmp_total = tmp_total + (select sum(val) from 
iot.stream_tmp),
-   tmp_count = tmp_total + (select count(*) from 
iot.stream_tmp);
+   set tmp_total = tmp_total + (select sum(val) from iot.stmp),
+   tmp_count = tmp_total + (select count(*) from iot.stmp);
+   delete from iot.stmp;
 end;
 
-iot.query('iot','collector');
+insert into stmp values('2005-09-23 12:34:26.736',1,12.34);
+select * from stmp;
+
+call iot.query('iot','collector');
 
 select * from iot.baskets();
 select * from iot.queries();
+
+call iot.activate();
+select * from tmp_aggregate;
diff --git a/sql/backends/monet5/iot/petrinet.c 
b/sql/backends/monet5/iot/petrinet.c
--- a/sql/backends/monet5/iot/petrinet.c
+++ b/sql/backends/monet5/iot/petrinet.c
@@ -150,7 +150,7 @@ PNregisterInternal(Client cntxt, MalBlkP
pnet[pnettop].modname = GDKstrdup(getModuleId(sig));
pnet[pnettop].fcnname = GDKstrdup(getFunctionId(sig));
snprintf(buf,IDLENGTH,"petri_%d",pnettop);
-   s = newFunction("iot", buf, FUNCTIONsymbol);
+   s = newFunction(iotRef, putName(buf,strlen(buf)), FUNCTIONsymbol);
nmb = s->def;
setArgType(nmb, nmb->stmt[0],0, TYPE_void);
 (void) newStmt(nmb, sqlRef, transactionRef);
@@ -304,6 +304,7 @@ PNexecute( void *n)
 {
PNnode *node= (PNnode *) n;
int i,j, idx;
+   str msg=  MAL_SUCCEED;
_DEBUG_PETRINET_ mnstr_printf(PNout, "#petrinet.execute 
%s.%s\n",node->modname, node->fcnname);
// first grab exclusive access to all streams.
MT_lock_set();
@@ -315,10 +316,10 @@ PNexecute( void *n)
 
_DEBUG_PETRINET_ mnstr_printf(PNout, "#petrinet.execute %s.%s all 
locked\n",node->modname, node->fcnname);
 
-   runMALsequence(mal_clients, node->mb, 1, 0, node->stk, 0, 0);
+   msg = runMALsequence(mal_clients, node->mb, 1, 0, node->stk, 0, 0);
node->status = PNPAUSED;
 
-   _DEBUG_PETRINET_ mnstr_printf(PNout, "#petrinet.execute %s.%s 
transition done\n",node->modname, node->fcnname);
+   _DEBUG_PETRINET_ mnstr_printf(PNout, "#petrinet.execute %s.%s 
transition done:%s\n",node->modname, node->fcnname, (msg != 
MAL_SUCCEED?msg:""));
 
MT_lock_set();
for ( i=0; i< j &&  node->enabled && node->places[i]; i++) {
@@ -330,7 +331,7 @@ PNexecute( void *n)
 }
 
 static void
-PNcontroller(void *dummy)
+PNscheduler(void *dummy)
 {
int idx = -1, i, j;
int k = -1;
@@ -339,6 +340,7 

MonetDB: iot - Added more data types for validation and cleaned ...

[Pedro Ferreira]

Changeset: 2b93ec7dc119 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2b93ec7dc119
Modified Files:
clients/iotclient/documentation/index.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streamscreator.py
Branch: iot
Log Message:

Added more data types for validation and cleaned code


diffs (truncated from 803 to 300 lines):

diff --git a/clients/iotclient/documentation/index.rst 
b/clients/iotclient/documentation/index.rst
--- a/clients/iotclient/documentation/index.rst
+++ b/clients/iotclient/documentation/index.rst
@@ -17,4 +17,5 @@ Contents:
iot_server_arguments.rst
restful_resources.rst
streams_data_types.rst
-  conclusion.rst
+   conclusion.rst
+
diff --git a/clients/iotclient/documentation/streams_data_types.rst 
b/clients/iotclient/documentation/streams_data_types.rst
--- a/clients/iotclient/documentation/streams_data_types.rst
+++ b/clients/iotclient/documentation/streams_data_types.rst
@@ -15,8 +15,8 @@ By default a column is not nullable, but
 Text Types
 ==
 
-Text, String, Character Large Object
-
+Text, String, CLOB, Character Large Object
+--
 
 MonetDB's string types with unbounded length. The insertion must be provided 
as a JSON string.
 
@@ -30,15 +30,37 @@ UUID
 
 An *Universally Unique Identifier* according to `RFC 4122 
`_. The insertion as a JSON string is 
validated against the regular expression 
:code:`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`.
 
+MAC
+---
+
+A *Media Access Control Address* identifier. The insertion as a JSON string is 
validated against the regular expression 
:code:`^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$`. As MonetDB doesn't have a 
MAC equivalent data type yet, the data is stored as :code:`char(17)`.
+
+URL
+---
+
+An *Uniform Resource Locator* as a specific type of an URI is validated 
according to `RFC 3987 `_. The insertion 
must also be a JSON String.
+
 Inet
 
 
 An *IPv4* address. The insertion as a JSON string is validated against the 
regular expression :code:`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$` with further 
semantic validation.
 
-URL

+InetSix
+---
 
-An *Uniform Resource Locator* as a specific type of an URI is validated 
according to `RFC 3987 `_. The insertion 
must also be a JSON String.
+An *IPv6* address. The value must be a JSON String. As MonetDB doesn't have an 
Ipv6 equivalent data type yet, the data is stored as :code:`char(45)`.
+
+.. warning:: Currently this validation is made with Python 
*socket.inet_pton()* function that is currently available only in some UNIX 
systems. `Check the documentation for further details 
`_.
+
+Regex
+-
+
+A string allways validated with a provided regular expression. The JSON must 
contain a :code:`regex` key with the regular expression. As MonetDB doesn't 
have a Regex equivalent data type yet, the data is stored as :code:`string`.
+
+Enum
+
+
+A SQL :code:`CHAR` type validated against a pre-defined array. During 
creation, the :code:`values` key must be present with a JSON array of Strings 
containing the values of the enum (ex: :code:`"values": ["red", "blue", 
"green"]`). The default value if provided must be one of the values.
 
 Number Types
 
@@ -90,22 +112,14 @@ As in numbers, a minimum and maximum val
 Date
 
 
-A regular date in the Gregorian Calendar, expressed in format "-MM-DD".
+A regular date in the Gregorian Calendar, expressed in format 
:code:`-MM-DD`.
 
 Time
 
 
-The time of day expressed in format "HH:MM:SS.sss".
+The time of day expressed in format :code:`HH:MM:SS.sss`. If the pair 
:code:`"timezone": false` is provided, the timezone will be truncated.
 
 Timestamp
 -
 
-A timestamp according to `RFC 3339 `_ 
with timezone. The regular expression is the standard `ISO 8601 with timezone 
format `_.
-
-Enum Type
-=
-
-Enum
-
-
-A SQL :code:`CHAR` type validated against a pre-defined array. During 
creation, the :code:`values` key must be present with a JSON array of Strings 
containing the values of the enum (ex: :code:`"values": ["red", "blue", 
"green"]`). The default value if provided must be one of the values.
+A timestamp according to `RFC 3339 `_ 
with timezone. The regular expression is the standard `ISO 8601 with timezone 
format `_. If the pair 
:code:`"timezone": false` is provided, the timezone will be truncated.
diff --git 

MonetDB: iot - Added stream information serialization using a JS...

[Pedro Ferreira]

Changeset: 6781d09b7830 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6781d09b7830
Added Files:
clients/iotclient/src/Streams/streamscreator.py
Removed Files:
clients/iotclient/src/Streams/semanticvalidation.py
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/flushing.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Utilities/filecreator.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

Added stream information serialization using a JSON config file


diffs (truncated from 710 to 300 lines):

diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -4,5 +4,5 @@ rfc3987==1.3.5
 strict-rfc3339==0.6
 python-dateutil==2.5.2
 pytz==2016.3
-pymonetdb==1.0
+pymonetdb==0.1.1
 tzlocal==1.2.2
diff --git a/clients/iotclient/src/Flask/app.py 
b/clients/iotclient/src/Flask/app.py
--- a/clients/iotclient/src/Flask/app.py
+++ b/clients/iotclient/src/Flask/app.py
@@ -1,7 +1,7 @@
 from flask import Flask
 from flask_restful import Api
 
-from restresources import StreamInput, StreamsInfo, StreamsHandling
+from restresources import StreamInput, StreamsInfo, StreamsHandling  # , 
ServerHandler
 
 
 def start_flask_iot_app(host, port):
@@ -21,4 +21,5 @@ def start_flask_admin_app(host, port):
 
 admin_api.add_resource(StreamsInfo, '/streams')
 admin_api.add_resource(StreamsHandling, '/context')
+#  admin_api.add_resource(ServerHandler, '/server')
 admin_app.run(host=host, port=port, threaded=True)
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
--- a/clients/iotclient/src/Flask/restresources.py
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -9,8 +9,18 @@ from tzlocal import get_localzone
 from Streams.jsonschemas import CREATE_STREAMS_SCHEMA, DELETE_STREAMS_SCHEMA
 from Streams.streamscontext import IOTStreamsException, IOTStreams
 
-Stream_context = IOTStreams()
-local_tz = get_localzone()  # for the correction of dates we must add the 
system's timezone
+Streams_Context = None
+Create_Streams_Validator = None
+Delete_Streams_Validator = None
+Local_Timezone = None
+
+
+def init_rest_resources():
+global Streams_Context, Create_Streams_Validator, 
Delete_Streams_Validator, Local_Timezone
+Local_Timezone = get_localzone()  # for the correction of dates we must 
add the system's timezone
+Create_Streams_Validator = Draft4Validator(CREATE_STREAMS_SCHEMA, 
format_checker=FormatChecker())
+Delete_Streams_Validator = Draft4Validator(DELETE_STREAMS_SCHEMA, 
format_checker=FormatChecker())
+Streams_Context = IOTStreams()
 
 
 class StreamInput(Resource):
@@ -18,22 +28,22 @@ class StreamInput(Resource):
 
 def get(self, schema_name, stream_name):  # check a single stream data
 try:  # check if stream exists, if not return 404
-stream = Stream_context.get_existing_stream(schema_name, 
stream_name)
-except IOTStreamsException as ex:
+stream = Streams_Context.get_existing_stream(schema_name, 
stream_name)
+except BaseException as ex:
 return ex.message, 404
-return stream.get_data_dictionary(), 200
+return stream.get_data_dictionary(include_number_tuples=True), 200
 
 def post(self, schema_name, stream_name):  # add data to a stream
-current_stamp = 
datetime.datetime.now(pytz.utc).astimezone(local_tz).isoformat()
+current_stamp = 
datetime.datetime.now(pytz.utc).astimezone(Local_Timezone).isoformat()
 
 try:  # check if stream exists, if not return 404
-stream = Stream_context.get_existing_stream(schema_name, 
stream_name)
-except IOTStreamsException as ex:
+stream = Streams_Context.get_existing_stream(schema_name, 
stream_name)
+except BaseException as ex:
 return ex.message, 404
 
 try:  # validate and insert data, if not return 400
 stream.validate_and_insert(json.loads(request.data), current_stamp)
-except Exception as ex:
+except BaseException as ex:
 return ex.message, 400
 return '', 201  # all ok, return 201
 
@@ -42,24 +52,21 @@ class StreamsInfo(Resource):
 """Collect all streams information"""
 
 def get(self):  # get all streams data
-return Stream_context.get_streams_data(), 200
+return Streams_Context.get_streams_data(), 200
 
 
 

MonetDB: iot - Merged with Martin code

[Pedro Ferreira]

Changeset: de1b527a4601 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=de1b527a4601
Added Files:
gdk/gdk_orderidx.c
monetdb5/modules/mal/Tests/orderidx00.malC
monetdb5/modules/mal/Tests/orderidx00.stable.err
monetdb5/modules/mal/Tests/orderidx00.stable.out
monetdb5/modules/mal/Tests/orderidx01.malC
monetdb5/modules/mal/Tests/orderidx01.stable.err
monetdb5/modules/mal/Tests/orderidx01.stable.out
monetdb5/modules/mal/Tests/orderidx02.malC
monetdb5/modules/mal/Tests/orderidx02.stable.err
monetdb5/modules/mal/Tests/orderidx02.stable.out
monetdb5/modules/mal/Tests/orderidx04.malC
monetdb5/modules/mal/orderidx.c
monetdb5/modules/mal/orderidx.h
monetdb5/modules/mal/orderidx.mal
sql/backends/monet5/sql_orderidx.c
sql/backends/monet5/sql_orderidx.h
sql/benchmarks/orderindex/experiment.sh
sql/benchmarks/tpch/Tests/lowcardinality.sql
sql/benchmarks/tpch/Tests/lowcardinality.stable.err
sql/benchmarks/tpch/Tests/lowcardinality.stable.out
sql/scripts/18_index.sql
sql/test/orderidx/Tests/All
sql/test/orderidx/Tests/simpletable.sql
sql/test/orderidx/Tests/simpletable.stable.err
sql/test/orderidx/Tests/simpletable.stable.out
sql/test/orderidx/Tests/smalltable.sql
sql/test/orderidx/Tests/smalltable.stable.err
sql/test/orderidx/Tests/smalltable.stable.out
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/exports.stable.out
gdk/Makefile.ag
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_hash.c
gdk/gdk_hash.h
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_private.h
gdk/gdk_search.c
gdk/gdk_select.c
gdk/gdk_storage.c
geom/monetdb5/geom.c
monetdb5/mal/Tests/tst611.malC
monetdb5/mal/mal.h
monetdb5/mal/mal_errors.h
monetdb5/mal/mal_resolve.c
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/Tests/All
monetdb5/modules/mal/Tests/inspect05.stable.out
monetdb5/modules/mal/Tests/inspect05.stable.out.int128
monetdb5/modules/mal/Tests/remote03.stable.err
monetdb5/modules/mal/mal_init.mal
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_upgrades.c
sql/benchmarks/tpch/Tests/All
sql/include/sql_catalog.h
sql/scripts/75_storagemodel.sql
sql/scripts/Makefile.ag
sql/server/sql_parser.y
sql/server/sql_scan.c
sql/storage/bat/bat_storage.c
sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.SQL.py
sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.stable.err
sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.stable.out
sql/test/BugTracker-2016/Tests/storagemodel.Bug-3923.stable.out
sql/test/BugTracker-2016/Tests/stream_table_crash.Bug-3952.sql
sql/test/BugTracker-2016/Tests/stream_table_crash.Bug-3952.stable.err
sql/test/BugTracker-2016/Tests/stream_table_crash.Bug-3952.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.err
sql/test/Dependencies/Tests/Dependencies.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out.int128
sql/test/Tests/median_stdev.stable.out
sql/test/Tests/outerref_in_selection.stable.out
sql/test/Tests/outerref_in_selection.stable.out.int128
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/Users/Tests/copyinto.stable.err
sql/test/Users/Tests/dropManyUsers.Bug-3764.stable.out
sql/test/Users/Tests/grantMonetdb.stable.err
sql/test/leaks/Tests/check0.stable.out
sql/test/leaks/Tests/check0.stable.out.int128
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.stable.out
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.stable.out
sql/test/leaks/Tests/check3.stable.out.int128
sql/test/leaks/Tests/check4.stable.out
sql/test/leaks/Tests/check4.stable.out.int128
sql/test/leaks/Tests/check5.stable.out
sql/test/leaks/Tests/check5.stable.out.int128
sql/test/leaks/Tests/drop3.stable.out
sql/test/leaks/Tests/drop3.stable.out.int128
sql/test/leaks/Tests/select1.stable.out
sql/test/leaks/Tests/select1.stable.out.int128

MonetDB: iot - Merged iot client with current DataCell kernel

[Pedro Ferreira]

Changeset: 4080899cd919 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4080899cd919
Added Files:

sql/test/BugTracker-2016/Tests/column_alias_in_where_clause.Bug-3947.stable.out.int128

sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out.32bit
sql/test/BugTracker-2016/Tests/epoch.Bug-3979.sql
sql/test/BugTracker-2016/Tests/epoch.Bug-3979.stable.err
sql/test/BugTracker-2016/Tests/epoch.Bug-3979.stable.out
sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.sql
sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.stable.err
sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.stable.out

sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.sql

sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.stable.err

sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.stable.out
sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.sql
sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.stable.err
sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.stable.out

sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.sql

sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.stable.err

sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.stable.out
sql/test/BugTracker-2016/Tests/rename_exps.Bug-3974.sql
sql/test/BugTracker-2016/Tests/rename_exps.Bug-3974.stable.err
sql/test/BugTracker-2016/Tests/rename_exps.Bug-3974.stable.out
sql/test/BugTracker-2016/Tests/subcorr-missing.Bug-3978.sql
sql/test/BugTracker-2016/Tests/subcorr-missing.Bug-3978.stable.err
sql/test/BugTracker-2016/Tests/subcorr-missing.Bug-3978.stable.out
sql/test/pg_regress/Tests/oid.stable.err.64bit.oid32
Removed Files:
sql/test/pg_regress/Tests/oid.stable.err.oid32
Modified Files:
MonetDB.spec
clients/Tests/All
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/malcheck.stable.out
clients/mapiclient/eventparser.c
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/odbc/driver/SQLBrowseConnect.c
clients/odbc/winsetup/setup.rc
debian/changelog
gdk/ChangeLog-Archive
gdk/gdk_bbp.c
gdk/gdk_hash.c
gdk/gdk_hash.h
gdk/gdk_heap.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_private.h
gdk/gdk_search.c
gdk/gdk_select.c
gdk/gdk_storage.c
gdk/gdk_utils.c
geom/BugTracker/Tests/All
geom/BugTracker/Tests/X_crash.SF-1971632.stable.err
geom/lib/libgeom.c
geom/lib/libgeom.h
geom/monetdb5/geom.c
geom/monetdb5/geom.h
geom/monetdb5/geomBulk.c
geom/sql/functions/Tests/ST_LineFromText.stable.err
geom/sql/functions/Tests/ST_MLineFromText.stable.err
geom/sql/functions/Tests/ST_MPointFromText.stable.err
geom/sql/functions/Tests/ST_MPolygonFromText.stable.err
geom/sql/functions/Tests/ST_PointFromText.stable.err
geom/sql/functions/Tests/ST_PolygonFromText.stable.err
geom/sql/functions/Tests/XYZ.stable.err
java/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
java/src/main/java/nl/cwi/monetdb/util/SQLExporter.java
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single
monetdb5/mal/mal_authorize.c
monetdb5/mal/mal_function.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/mtime.mal
monetdb5/optimizer/opt_iot.c
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
sql/backends/monet5/iot/Tests/iot00.sql
sql/backends/monet5/iot/basket.c
sql/backends/monet5/iot/basket.h
sql/backends/monet5/iot/basket.mal
sql/backends/monet5/iot/iot.c
sql/backends/monet5/iot/iot.h
sql/backends/monet5/iot/petrinet.c
sql/backends/monet5/iot/petrinet.h
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_aggr_bte.mal
sql/backends/monet5/sql_aggr_dbl.mal
sql/backends/monet5/sql_aggr_flt.mal
sql/backends/monet5/sql_aggr_hge.mal
sql/backends/monet5/sql_aggr_int.mal
sql/backends/monet5/sql_aggr_lng.mal
sql/backends/monet5/sql_aggr_sht.mal
sql/backends/monet5/sql_aggr_wrd.mal
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_optimizer.c

MonetDB: iot - First version of IOT webserver added

[Pedro Ferreira]

Changeset: 2d1c8b51a218 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2d1c8b51a218
Added Files:
clients/iotclient/README
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/__init__.py
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/__init__.py
clients/iotclient/src/Settings/filesystem.py
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/__init__.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/flushing.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/semanticvalidation.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Utilities/__init__.py
clients/iotclient/src/Utilities/filecreator.py
clients/iotclient/src/Utilities/numberutilities.py
clients/iotclient/src/Utilities/readwritelock.py
clients/iotclient/src/__init__.py
clients/iotclient/src/main.py
Branch: iot
Log Message:

First version of IOT webserver added


diffs (truncated from 1667 to 300 lines):

diff --git a/clients/iotclient/README b/clients/iotclient/README
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/README
@@ -0,0 +1,30 @@
+MonetDB IOT RESTful WebServer to use with the new DataCell engine written in 
Python 2 using Flask-RESTful framework.
+
+The program is composed by two webservers: one for administration purposes 
regarding the creation and deletion of streams, and other for stream input 
using JSON requests.
+
+
+REST resources available on administration webserver running by default on 
port 8001 (should be privated from outside networks):
+
+/streams - GET - Returns JSON file with details about all the streams 
currently supported on the webserver.
+
+/context - POST - Creates a stream using the JSON Schema defined on 
src/Streams/jsonschemas.py on CREATE_STREAMS_SCHEMA variable.
+
+/context - DELETE - Deletes a stream using the JSON Schema defined on 
src/Streams/jsonschemas.py on DELETE_STREAMS_SCHEMA variable.
+
+
+REST resources available on iot webserver running by default on port 8000:
+
+/streams - GET - Returns JSON file with details about all the streams 
currently supported on the webserver.
+
+/stream// - GET - Returns JSON file 
with details of the stream_name on schema_name
+
+/stream// - POST - Inserts data to the 
stream using a JSON schema generated while creating the stream. The input must 
be a JSON array even for a single input.
+
+
+Currently supports most of MonetDB datatypes. An implicit Timestamp column is 
added with the tuples arrival date. The flushing of the baskets can be either 
Tuple or Time based, as specied on the streams' creation.
+
+More details coming soon.
+
+
+Maintainer: Pedro Ferreira at CWI, email: p.e.ferre...@cwi.nl
+
diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/requirements.txt
@@ -0,0 +1,8 @@
+Flask-RESTful==0.3.5
+jsonschema==2.5.1
+rfc3987==1.3.5
+strict-rfc3339==0.6
+python-dateutil==2.5.2
+pytz==2016.3
+pymonetdb==1.0
+tzlocal==1.2.2
diff --git a/clients/iotclient/src/Flask/__init__.py 
b/clients/iotclient/src/Flask/__init__.py
new file mode 100644
diff --git a/clients/iotclient/src/Flask/app.py 
b/clients/iotclient/src/Flask/app.py
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/src/Flask/app.py
@@ -0,0 +1,24 @@
+from flask import Flask
+from flask_restful import Api
+
+from restresources import StreamInput, StreamsInfo, StreamsHandling
+
+
+def start_flask_iot_app(host, port):
+iot_app = Flask(__name__)
+iot_app.config['BUNDLE_ERRORS'] = True
+iot_api = Api(iot_app)
+
+iot_api.add_resource(StreamsInfo, '/streams')
+iot_api.add_resource(StreamInput, 
'/stream//')
+iot_app.run(host=host, port=port, threaded=True)
+
+
+def start_flask_admin_app(host, port):
+admin_app = Flask(__name__)
+admin_app.config['BUNDLE_ERRORS'] = True
+admin_api = Api(admin_app)
+
+admin_api.add_resource(StreamsInfo, '/streams')
+admin_api.add_resource(StreamsHandling, '/context')
+admin_app.run(host=host, port=port, threaded=True)
diff --git a/clients/iotclient/src/Flask/restresources.py 
b/clients/iotclient/src/Flask/restresources.py
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/src/Flask/restresources.py
@@ -0,0 +1,78 @@
+import datetime
+import json
+import pytz
+
+from flask import request
+from flask_restful import Resource
+from jsonschema import Draft4Validator, FormatChecker
+from tzlocal import get_localzone
+from src.Streams.jsonschemas import CREATE_STREAMS_SCHEMA, 
DELETE_STREAMS_SCHEMA
+from src.Streams.streamscontext import IOTStreamsException, IOTStreams

MonetDB: iot - Finished documentation and corrected regular expr...

[Pedro Ferreira]

Changeset: 5052d18264ac for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5052d18264ac
Added Files:
clients/iotclient/documentation/conclusion.rst
Removed Files:
clients/iotclient/documentation/future_work.rst
clients/iotclient/documentation/streams_creation.rst
Modified Files:
clients/iotclient/documentation/index.rst
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/documentation/streams_data_types.rst
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
Branch: iot
Log Message:

Finished documentation and corrected regular expressions


diffs (truncated from 432 to 300 lines):

diff --git a/clients/iotclient/documentation/conclusion.rst 
b/clients/iotclient/documentation/conclusion.rst
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/documentation/conclusion.rst
@@ -0,0 +1,25 @@
+.. _future_work:
+
+**
+Final Acknowledgements
+**
+
+Future Work
+===
+
+Currently the project is on a testing phase. More data types will also be 
added.
+
+Soon another web server will be built to implement an API for the baskets 
outputs from DataCell.
+
+.. _about-the-project:
+
+About the Project
+=
+
+.. _Send questions and suggestions to e-mail.: p.e.ferre...@cwi.nl
+
+Project maintained by Pedro Ferreira at CWI. `Send questions and suggestions 
to e-mail.`_
+
+MonetDB source repository: https://dev.monetdb.org/hg/MonetDB
+
+MonetDB solutions: https://monetdbsolutions.com/
diff --git a/clients/iotclient/documentation/future_work.rst 
b/clients/iotclient/documentation/future_work.rst
deleted file mode 100644
--- a/clients/iotclient/documentation/future_work.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _future_work:
-
-
-**
-Other information
-**
-
-.. _future-work:
-
-Future Work
-===
-
-Talk about it
-
-.. _about-the-project:
-
-About the Project
-=
-
-IOT stuff
diff --git a/clients/iotclient/documentation/index.rst 
b/clients/iotclient/documentation/index.rst
--- a/clients/iotclient/documentation/index.rst
+++ b/clients/iotclient/documentation/index.rst
@@ -16,7 +16,5 @@ Contents:
introduction.rst
iot_server_arguments.rst
restful_resources.rst
-   streams_creation.rst
streams_data_types.rst
-   future_work.rst
-
+  conclusion.rst
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -53,7 +53,7 @@ Custom name of the host. By default is t
 Web Server Listening
 
 
-For security purposes, two web servers are created by the application. On the 
administration server are present operations to create and delete streams, 
while on application server the insert operations are present instead. The 
administration server should be listening on the host only, while the 
application server should be listening to the outside.
+For security purposes, two web servers are created by the application. On the 
administration server are present operations to create and delete streams, 
while on application server the insert operations are present instead. The 
administration server should be listening on the host only, while the 
application server should be listening to all interfaces.
 
 **-ih  - -ihost=**
 
@@ -94,5 +94,3 @@ Name of the user to authenticate. By def
 **-dd  - -ddatabase=**
 
 Name of database to use. By default is :code:`iotdb` database.
-
-
diff --git a/clients/iotclient/documentation/restful_resources.rst 
b/clients/iotclient/documentation/restful_resources.rst
--- a/clients/iotclient/documentation/restful_resources.rst
+++ b/clients/iotclient/documentation/restful_resources.rst
@@ -5,11 +5,125 @@
 RESTful Resources
 *
 
-Introduction
+Bellow is lested the available RESTful resource for both servers on IOT 
application. In case of error, the server will return the error message with 
the respective HTTP response code. 
+
+.. important:: All the keywords provided on JSON must be lower case, as well 
the column data types.
+
+Administration Server
+=
+
+The administration server provides resources to create and delete streams. 
Should be listening on the host only.
 
 .. _streams:
 
 /streams
-=
+
 
-Talk about this and the others
+**GET**
+
+Returns a JSON file with details about all the streams currently created on 
the webserver. For each stream besides its schema and name, it provides the 
currently number of tuples inserted on the baskets per column, description of 
columns (`See data types <streams_data_types.html#data_types>`__), the flushing 
method (`See streams cr

MonetDB: iot - Web api working with iot server. Started adding d...

[Pedro Ferreira]

WebAPI's documentation!
+==
 
 Contents:
 
@@ -14,7 +14,6 @@ Contents:
:caption: Table of Contents
 
introduction.rst
-   iot_server_arguments.rst
-   restful_resources.rst
-   streams_data_types.rst
+   api_server_arguments.rst
+   websockets_api.rst
conclusion.rst
diff --git a/clients/iotclient/documentation/introduction.rst 
b/clients/iotapi/documentation/introduction.rst
copy from clients/iotclient/documentation/introduction.rst
copy to clients/iotapi/documentation/introduction.rst
--- a/clients/iotclient/documentation/introduction.rst
+++ b/clients/iotapi/documentation/introduction.rst
@@ -4,10 +4,10 @@
 Introduction
 
 
-This application is a Python web server destined to extend MonetDB RDBMS to 
the Internet-of-Things while using a streaming engine. This project is derived 
from Pedro Ferreira master thesis, a junior researcher at CWI. The objective of 
the master thesis to re-evaluate the DataCell extension of MonetDB to the IOT 
world, which was postponed for several years since its release. To accomplish 
this, an IOT topology around MonetDB is being built, where this server aims to 
analyse the input of streams. Later it will also be included another web server 
to analyze the output of streams.
+This application is a Python web server destined to extend MonetDB RDBMS to 
the Internet-of-Things while using a streaming engine. This project is derived 
from Pedro Ferreira master thesis, a junior researcher at CWI. The objective of 
the master thesis to build a streaming extension of MonetDB to the IOT world, 
which is obtaining much demand nowadays. To accomplish this, an IOT topology 
around MonetDB is being built, where this server aims to analyse the input of 
streams. At the same time, another web server is being built to analyze the 
streams' output.
 
-This web server is built using Python programming language, version 2.7, using 
`Flask-RESTful <https://pypi.python.org/pypi/Flask-RESTful>`_ framework. The 
required packages on requirements.txt should be installed before running the 
server. 
+This web server is built using Python programming language, version 2.7, using 
a `Simple Websockets server 
<https://github.com/dpallot/simple-websocket-server>`_ . The required packages 
on requirements.txt should be installed before running the server.
 
-The web server is capable of creating and deleting streams for the renewed 
DataCell extension. After a stream is created, it's possible to make batch 
inserts on it. The requests are made using RESTful requests with JSON content. 
Both stream creation and stream insertion are validated using `JSON Schema 
Draft 4 <http://json-schema.org/documentation.html>`_. On a batch insert if a 
tuple is invalid, then no tuples are inserted. The stream engine adds an 
implicit timestamp column to recognize when the tuple was inserted. The 
inserted tuples are later evaluated using MonetDB's relational engine with 
continuous queries.
+The server creates a polling connection with MonetDB, listening for every new 
output queries. The Streaming context will then be constantly updated during 
the polling connection. A small publisher/subscriber pattern was added to the 
server where the clients can to subscribe to notified right away when an output 
basket is created. It is also possible to perform small queries on the created 
data for pagination while including an offset or a limit of the number of 
tuples to retrieve. All the communication process is assured using a 
full-duplex WebSockets connection.
 
-The following chapters explain the server parameters, RESTful resources and 
how to manage the streams.
+The following chapters explain the server parameters and the Websockets 
protocol used.
diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
new file mode 100644
--- /dev/null
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -0,0 +1,68 @@
+.. _websockets_api:
+
+**
+WebSockets API
+**
+
+Bellow is listed the available request and response messages for the 
WebSockets API.
+
+Requests
+
+
+The client must always provide a JSON string in a request with a request field 
indicating the intended action to perform in the server, followed by the other 
specific fields depending on the request.
+
+.. important:: The :code:`request` field on the JSON request must always be 
lowercase!
+
+The following sections explain the available :code:`request` fields.
+
+sub
+---
+
+Subscribes for new basket creations from a specific stream. Whenever a basket 
is created, the server sends a notification message indicating the number of 
inserted tuples in the new basket. The user has to specify the stream's name 
and schema.
+
+.. code-block:: json
+
+{
+"request": "sub",
+"schema": "measures",
+"stream": "temperature&

MonetDB: iot - Fixed heartbeats validation, cleaned some spaces

[Pedro Ferreira]

Changeset: 7d4e342fbc94 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7d4e342fbc94
Modified Files:
sql/backends/monet5/iot/50_iot.sql
sql/backends/monet5/iot/Tests/ais/ais01.sql
sql/backends/monet5/iot/petrinet.c
Branch: iot
Log Message:

Fixed heartbeats validation, cleaned some spaces


diffs (57 lines):

diff --git a/sql/backends/monet5/iot/50_iot.sql 
b/sql/backends/monet5/iot/50_iot.sql
--- a/sql/backends/monet5/iot/50_iot.sql
+++ b/sql/backends/monet5/iot/50_iot.sql
@@ -66,10 +66,10 @@ create procedure iot.export("schema" str
 -- input/output places
 create procedure iot.receptor("schema" string, "table" string, dir string)
external name iot.receptor;
+
 create procedure iot.emitter("schema" string, "table" string, dir string)
external name iot.emitter;
 
-
 create procedure iot.heartbeat("schema" string, "table" string, msec integer)
external name iot.heartbeat;
 
@@ -84,7 +84,6 @@ create procedure iot.cycles("schema" str
external name iot.cycles;
 
 -- Inspection tables
-
 create function iot.gettumble("schema" string, "table" string) returns integer
 external name iot.gettumble;
 
@@ -118,7 +117,6 @@ returns table( "table" string, error str
 external name iot.errors;
 
 -- tables for iotwebserver
-
 CREATE TABLE iot.webserverstreams (table_id INTEGER, base TINYINT, "interval" 
INTEGER NULL, unit CHAR(1) NULL);
 
 CREATE TABLE iot.webservercolumns (column_id INTEGER, special TINYINT NULL, 
validation1 STRING NULL, validation2 STRING NULL);
diff --git a/sql/backends/monet5/iot/Tests/ais/ais01.sql 
b/sql/backends/monet5/iot/Tests/ais/ais01.sql
--- a/sql/backends/monet5/iot/Tests/ais/ais01.sql
+++ b/sql/backends/monet5/iot/Tests/ais/ais01.sql
@@ -11,7 +11,7 @@ INSERT INTO iot.webserverstreams
SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
 
 -- We don't set the tumbling, so no tuple will be reused in the following 
window
-CALL iot.heartbeat('ais', 'vessels', 8000); 
+CALL iot.heartbeat('ais', 'vessels', 8000);
 
 --Q1 Calculate speed of ships per hour (in knots) -- Stream only
 
diff --git a/sql/backends/monet5/iot/petrinet.c 
b/sql/backends/monet5/iot/petrinet.c
--- a/sql/backends/monet5/iot/petrinet.c
+++ b/sql/backends/monet5/iot/petrinet.c
@@ -107,8 +107,8 @@ PNheartbeat(str mod, str fcn, int ticks)
 {
int i;
 
-   if (ticks <= 0)
-   throw(MAL,"iot.heartbeat","The heartbeat should be > 0\n");
+   if (ticks < 0)
+   throw(MAL,"iot.heartbeat","The heartbeat should be >= 0\n");
 
for(i = 0; i < pnettop; i++) {
if(strcmp(pnet[i].modname,mod) == 0 && 
strcmp(pnet[i].fcnname,fcn) == 0) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Fixed baskets initialization, cleaned more error ...

[Pedro Ferreira]

Changeset: f25257924841 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f25257924841
Modified Files:
sql/backends/monet5/iot/basket.c
Branch: iot
Log Message:

Fixed baskets initialization, cleaned more error messages


diffs (137 lines):

diff --git a/sql/backends/monet5/iot/basket.c b/sql/backends/monet5/iot/basket.c
--- a/sql/backends/monet5/iot/basket.c
+++ b/sql/backends/monet5/iot/basket.c
@@ -49,25 +49,33 @@ static int bsktTop = 0, bsktLimit = 0;
 // Find an empty slot in the basket catalog
 static int BSKTnewEntry(void)
 {
-   int i;
+   int i = bsktTop;
if (bsktLimit == 0) {
bsktLimit = MAXBSKT;
baskets = (BasketRec *) GDKzalloc(bsktLimit * 
sizeof(BasketRec));
+   for (i = 0; i < bsktLimit; i++) { /* init the new entries as 
null */
+   baskets[i].table_name == NULL;
+   }
bsktTop = 1; /* entry 0 is used as non-initialized */
-   } else if (bsktTop +1 == bsktLimit) {
+   } else if (bsktTop + 1 == bsktLimit) {
bsktLimit += MAXBSKT;
baskets = (BasketRec *) GDKrealloc(baskets, bsktLimit * 
sizeof(BasketRec));
+   for (i = bsktTop + 1; i < bsktLimit; i++) { /* init the new 
entries as null */
+   baskets[i].table_name == NULL;
+   }
}
-   for (i = 1; i < bsktLimit; i++)
+   
+   for (i = 1; i < bsktLimit; i++) { /* find an available slot */
if (baskets[i].table_name == NULL)
break;
-   MT_lock_init([i].lock,"bsktlock");
-
-   bsktTop++;
+   }
+   if(i >= bsktTop) { /* if it's the last one we need to increment bsktTop 
*/
+   bsktTop++;
+   }
+   MT_lock_init([i].lock,"bsktlock");  
return i;
 }
 
-
 // free all malloced space
 void
 BSKTclean(int idx)
@@ -141,13 +149,13 @@ BSKTnewbasket(mvc *m, sql_schema *s, sql
 
 if ( !(tpe <= TYPE_str || tpe == TYPE_date || tpe == TYPE_daytime || 
tpe == TYPE_timestamp) ){
MT_lock_unset();
-   throw(MAL,"baskets.register","Unsupported type %d",tpe);
+   throw(MAL,"baskets.register","Unsupported type 
%d\n",tpe);
}
colcnt++;
}
if( colcnt == MAXCOLS){
BSKTclean(idx);
-   throw(MAL,"baskets.register","too many columns");
+   throw(MAL,"baskets.register","Too many columns\n");
}
 
// collect the column names and the storage
@@ -192,11 +200,11 @@ BSKTregisterInternal(Client cntxt, MalBl
 
s = mvc_bind_schema(m, sch);
if (s == NULL)
-   throw(SQL, "iot.register", "Schema missing");
+   throw(SQL, "iot.register", "Schema missing\n");
 
t = mvc_bind_table(m, s, tbl);
if (t == NULL)
-   throw(SQL, "iot.register", "Table missing '%s'", tbl);
+   throw(SQL, "iot.register", "Table missing '%s'\n", tbl);
 
msg=  BSKTnewbasket(m, s, t);
return msg;
@@ -231,7 +239,7 @@ BSKTheartbeat(Client cntxt, MalBlkPtr mb
(void) mb;
 
if( ticks < 0)
-   throw(SQL,"basket.heartbeat","Positive heartbeat expected]n");
+   throw(SQL,"basket.heartbeat","Positive heartbeat expected\n");
idx = BSKTlocate(sch, tbl);
if( idx == 0)
return PNheartbeat(sch,tbl,ticks);
@@ -272,7 +280,7 @@ BSKTwindow(Client cntxt, MalBlkPtr mb, M
(void) cntxt;
(void) mb;
if( elm <= 0)
-   throw(SQL,"basket.window","Positive slice expected]n");
+   throw(SQL,"basket.window","Positive slice expected\n");
idx = BSKTlocate(sch, tbl);
if( idx == 0){
BSKTregisterInternal(cntxt, mb, sch, tbl);
@@ -332,7 +340,7 @@ BSKTtid(Client cntxt, MalBlkPtr mb, MalS
 
bskt = BSKTlocate(sch,tbl);
if( bskt == 0)  
-   throw(SQL,"basket.bind","Stream table column '%s.%s' not 
found",sch,tbl);
+   throw(SQL,"basket.bind","Stream table column '%s.%s' not 
found\n",sch,tbl);
b = baskets[bskt].bats[0];
 
 tids = BATnew(TYPE_void, TYPE_void, 0, TRANSIENT);
@@ -369,7 +377,7 @@ BSKTbind(Client cntxt, MalBlkPtr mb, Mal
VIEWbounds(b,bn, 0, 
baskets[bskt].winsize);
BBPkeepref(*ret =  bn->batCacheid);
} else
-   throw(SQL,"iot.bind","Can not create 
view %s.%s.%s["BUNFMT"]",sch,tbl,col,baskets[bskt].winsize );
+   throw(SQL,"iot.bind","Can not create 
view %s.%s.%s["BUNFMT"]\n",sch,tbl,col,baskets[bskt].winsize );
} else{
BBPkeepref( *ret = b->batCacheid);
BBPfix(b->batCacheid); // don't loose it

MonetDB: iot - Fixed MAL comments and an AIS test

[Pedro Ferreira]

Changeset: 31bb025dd20d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=31bb025dd20d
Modified Files:
sql/backends/monet5/iot/50_iot.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
sql/backends/monet5/iot/basket.mal
sql/backends/monet5/iot/iot.mal
sql/backends/monet5/iot/petrinet.mal
Branch: iot
Log Message:

Fixed MAL comments and an AIS test


diffs (118 lines):

diff --git a/sql/backends/monet5/iot/50_iot.sql 
b/sql/backends/monet5/iot/50_iot.sql
--- a/sql/backends/monet5/iot/50_iot.sql
+++ b/sql/backends/monet5/iot/50_iot.sql
@@ -74,6 +74,7 @@ create procedure iot.emitter("schema" st
 
 create procedure iot.heartbeat("schema" string, "table" string, msec integer)
external name iot.heartbeat;
+
 create procedure iot.heartbeat("schema" string, "table" string, msec bigint)
external name iot.heartbeat;
 
diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
@@ -21,7 +21,7 @@ CREATE STREAM TABLE stations (implicit_t
 -- Inserts for iot web server (providing time based flush of 8 seconds)
 INSERT INTO iot.webserverstreams SELECT tabl.id, 2 , 8, 's' FROM sys.tables 
tabl INNER JOIN sys.schemas sch ON tabl.schema_id = sch.id WHERE tabl.name = 
'vessels6' AND sch.name = 'ais';
 
-INSERT INTO iot.webserverstreams SELECT tabl.id, 2 , 10, 's' FROM sys.tables 
tabl INNER JOIN sys.schemas sch ON tabl.schema_id = sch.id WHERE tabl.name = 
'stations' AND sch.name = 'ais';
+INSERT INTO iot.webserverstreams SELECT tabl.id, 2 , 8, 's' FROM sys.tables 
tabl INNER JOIN sys.schemas sch ON tabl.schema_id = sch.id WHERE tabl.name = 
'stations' AND sch.name = 'ais';
 
 --Q6 For each station calulate ship within a radios of 3 km -- Stream join
 
@@ -41,7 +41,7 @@ CALL iot.query('ais', 'ais06q');
 CALL iot.pause();
 -- We don't set the tumbling, so no tuple will be reused in the following 
window
 CALL iot.heartbeat('ais', 'vessels6', 8000);
-CALL iot.heartbeat('ais', 'stations', 1);
+CALL iot.heartbeat('ais', 'stations', 8000);
 CALL iot.resume();
 
 CALL iot.pause();
diff --git a/sql/backends/monet5/iot/basket.mal 
b/sql/backends/monet5/iot/basket.mal
--- a/sql/backends/monet5/iot/basket.mal
+++ b/sql/backends/monet5/iot/basket.mal
@@ -28,7 +28,7 @@ comment "Collect the candidates for a ba
 
 pattern bind(mvc:int,sch:str,tbl:str,col:str):bat[:any]
 address BSKTbind
-comment "access the stream basket column";
+comment "Access the stream basket column";
 
 pattern append(mvc:int, sch:str, tbl:str, col:str, val:any):int
 address BSKTappend
@@ -47,7 +47,7 @@ address mvc_clear_table_wrap;
 
 pattern tumble(mvc:any, sch:str, tbl:str):int
 address BSKTtumble
-comment "Make the basket available for the scheduler";
+comment "Apply tumbling to the basket";
 
 pattern settumble(sch:str, tbl:str, n:int):void
 address BSKTsettumble
diff --git a/sql/backends/monet5/iot/iot.mal b/sql/backends/monet5/iot/iot.mal
--- a/sql/backends/monet5/iot/iot.mal
+++ b/sql/backends/monet5/iot/iot.mal
@@ -67,7 +67,7 @@ comment "Return a table the erroneous ev
 
 command error(sch:str,fcn:str,msg:str)
 address BSKTerror
-comment "Save exception message ";
+comment "Save an exception message";
 
 pattern iot.heartbeat(sch:str,tbl:str,N:int)
 address BSKTheartbeat
@@ -83,11 +83,11 @@ comment "Get the hearbeat delay";
 
 pattern iot.tumble(sch:str,tbl:str,N:int)
 address BSKTsettumble
-comment "Set the window tumbling at most N events from the stream ";
+comment "Set the window tumbling at most N events from the stream";
 
 pattern iot.gettumble(sch:str,tbl:str):int
 address BSKTgettumble
-comment "Get the window tumbling size orf the stream ";
+comment "Get the window tumbling size of the stream";
 
 unsafe pattern iot.window(sch:str,tbl:str,N:int)
 address BSKTwindow
@@ -95,7 +95,7 @@ comment "Use a window of precisely N eve
 
 pattern iot.getwindow(sch:str,tbl:str):int
 address BSKTgetwindow
-comment "Set the window tumbling at most N events from the stream ";
+comment "Set the window tumbling at most N events from the stream";
 
 pattern iot.show(sch:str,tbl:str)
 address PNshow
diff --git a/sql/backends/monet5/iot/petrinet.mal 
b/sql/backends/monet5/iot/petrinet.mal
--- a/sql/backends/monet5/iot/petrinet.mal
+++ b/sql/backends/monet5/iot/petrinet.mal
@@ -22,7 +22,7 @@ address PNregister;
 pattern register(mod:str, fcn:str)
 address PNregister
 comment "Add a continous MAL query to the Petri net. It will analyse
-the MAL block to determine the input/output dependencies. ";
+the MAL block to determine the input/output dependencies.";
 
 pattern resume(mod:str, fcn:str)
 address PNresume
@@ -30,7 +30,7 @@ comment "Activate a continuous query";
 
 pattern resume()
 address PNresume
-comment "Activate all continuous query";
+comment "Activate all continuous queries";
 
 pattern pause(mod:str, fcn:str)
 

MonetDB: iot - Updated ais tests

[Pedro Ferreira]

Changeset: e0e57cb5831d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e0e57cb5831d
Modified Files:
clients/iotclient/requirements.txt
sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais03.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais04.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais05.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais08.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais09.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
Branch: iot
Log Message:

Updated ais tests


diffs (truncated from 477 to 300 lines):

diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,4 +1,4 @@
-fake-factory>=0.5.8
+fake-factory>=0.5.9
 Flask-RESTful>=0.3.5
 IPy>=0.83
 jsonschema>=2.5.1
diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
@@ -3,12 +3,11 @@ SET SCHEMA ais;
 SET optimizer = 'iot_pipe';
 
 -- Vessels positions reports table based on AIS messages types 1, 2 and 3
-CREATE STREAM TABLE vessels (implicit_timestamp timestamp, mmsi int, lat real, 
lon real, nav_status tinyint, sog real, rotais smallint);
+CREATE STREAM TABLE vessels1 (implicit_timestamp timestamp, mmsi int, lat 
real, lon real, nav_status tinyint, sog real, rotais smallint);
 
 -- Position reports are sent every 3-5 seconds so is resonable to consume the 
tuples arrived on the last 8 seconds
 -- Inserts for iot web server (providing time based flush of 8 seconds)
-INSERT INTO iot.webserverstreams
-   SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
+INSERT INTO iot.webserverstreams SELECT tabl.id, 2 , 8, 's' FROM sys.tables 
tabl INNER JOIN sys.schemas sch ON tabl.schema_id = sch.id WHERE tabl.name = 
'vessels1' AND sch.name = 'ais';
 
 --Q1 Calculate speed of ships per hour (in knots) -- Stream only
 
@@ -18,12 +17,18 @@ CREATE PROCEDURE ais01q()
 BEGIN  
INSERT INTO ais01r
WITH data_time AS (SELECT current_timestamp AS cur_time)
-   SELECT cur_time, mmsi, sog FROM vessels CROSS JOIN data_time 
WHERE (implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels GROUP BY mmsi);
+   SELECT cur_time, mmsi, sog FROM vessels1 CROSS JOIN data_time 
WHERE (implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels1 GROUP BY mmsi);
 END;
 
 CALL iot.query('ais', 'ais01q');
 CALL iot.pause();
 -- We don't set the tumbling, so no tuple will be reused in the following 
window
-CALL iot.heartbeat('ais', 'vessels', 8000);
+CALL iot.heartbeat('ais', 'vessels1', 8000);
 CALL iot.resume();
 
+CALL iot.pause();
+DELETE FROM iot.webserverstreams;
+DROP PROCEDURE ais01q;
+DROP TABLE vessels1;
+DROP TABLE ais01r;
+
diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
@@ -2,12 +2,11 @@ SET SCHEMA ais;
 SET optimizer = 'iot_pipe';
 
 -- Vessels positions reports table based on AIS messages types 1, 2 and 3
-CREATE STREAM TABLE vessels (implicit_timestamp timestamp, mmsi int, lat real, 
lon real, nav_status tinyint, sog real, rotais smallint);
+CREATE STREAM TABLE vessels2 (implicit_timestamp timestamp, mmsi int, lat 
real, lon real, nav_status tinyint, sog real, rotais smallint);
 
 -- Position reports are sent every 3-5 seconds so is resonable to consume the 
tuples arrived on the last 8 seconds
 -- Inserts for iot web server (providing time based flush of 8 seconds)
-INSERT INTO iot.webserverstreams
-   SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
+INSERT INTO iot.webserverstreams SELECT tabl.id, 2 , 8, 's' FROM sys.tables 
tabl INNER JOIN sys.schemas sch ON tabl.schema_id = sch.id WHERE tabl.name = 
'vessels2' AND sch.name = 'ais';
 
 --Q2 Number of ship per hour -- Stream only
 
@@ -16,12 +15,18 @@ CREATE TABLE ais02r (calc_time timestamp
 CREATE PROCEDURE ais02q()
 BEGIN
INSERT INTO ais02r
-   SELECT current_timestamp, count(DISTINCT mmsi) FROM vessels;
+   SELECT current_timestamp, count(DISTINCT mmsi) FROM vessels2;
 END;
 
 CALL iot.query('ais', 'ais02q');
 CALL iot.pause();
 -- We don't set the 

MonetDB: iot - Updated ais tests

[Pedro Ferreira]

Changeset: 975f76aedd79 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=975f76aedd79
Modified Files:
sql/backends/monet5/iot/Tests/ais/ais01.sql
sql/backends/monet5/iot/Tests/ais/ais02.sql
sql/backends/monet5/iot/Tests/ais/ais03.sql
sql/backends/monet5/iot/Tests/ais/ais04.sql
sql/backends/monet5/iot/Tests/ais/ais05.sql
sql/backends/monet5/iot/Tests/ais/ais06.sql
sql/backends/monet5/iot/Tests/ais/ais07.sql
sql/backends/monet5/iot/Tests/ais/ais08.sql
sql/backends/monet5/iot/Tests/ais/ais09.sql
sql/backends/monet5/iot/Tests/ais/ais10.sql
sql/backends/monet5/iot/Tests/ais/ais11.sql
Branch: iot
Log Message:

Updated ais tests


diffs (truncated from 358 to 300 lines):

diff --git a/sql/backends/monet5/iot/Tests/ais/ais01.sql 
b/sql/backends/monet5/iot/Tests/ais/ais01.sql
--- a/sql/backends/monet5/iot/Tests/ais/ais01.sql
+++ b/sql/backends/monet5/iot/Tests/ais/ais01.sql
@@ -10,18 +10,20 @@ CREATE STREAM TABLE vessels (implicit_ti
 INSERT INTO iot.webserverstreams
SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
 
--- We don't set the tumbling, so no tuple will be reused in the following 
window
-CALL iot.heartbeat('ais', 'vessels', 8000);
-
 --Q1 Calculate speed of ships per hour (in knots) -- Stream only
 
-CREATE STREAM TABLE ais01r (calc_time timestamp, mmsi int, sog real);
+CREATE TABLE ais01r (calc_time timestamp, mmsi int, sog real);
 
 CREATE PROCEDURE ais01q()
-BEGIN
+BEGIN  
INSERT INTO ais01r
-   SELECT current_timestamp, mmsi, sog FROM vessels WHERE 
(implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels GROUP BY mmsi);
+   WITH data_time AS (SELECT current_timestamp AS cur_time)
+   SELECT cur_time, mmsi, sog FROM vessels CROSS JOIN data_time 
WHERE (implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels GROUP BY mmsi);
 END;
 
 CALL iot.query('ais', 'ais01q');
+CALL iot.pause();
+-- We don't set the tumbling, so no tuple will be reused in the following 
window
+CALL iot.heartbeat('ais', 'vessels', 8000);
+CALL iot.resume();
 
diff --git a/sql/backends/monet5/iot/Tests/ais/ais02.sql 
b/sql/backends/monet5/iot/Tests/ais/ais02.sql
--- a/sql/backends/monet5/iot/Tests/ais/ais02.sql
+++ b/sql/backends/monet5/iot/Tests/ais/ais02.sql
@@ -10,12 +10,9 @@ CREATE STREAM TABLE vessels (implicit_ti
 INSERT INTO iot.webserverstreams
SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
 
--- We don't set the tumbling, so no tuple will be reused in the following 
window
-CALL iot.heartbeat('ais', 'vessels', 8000); 
-
 --Q2 Number of ship per hour -- Stream only
 
-CREATE STREAM TABLE ais02r (calc_time timestamp, number_ships int);
+CREATE TABLE ais02r (calc_time timestamp, number_ships int);
 
 CREATE PROCEDURE ais02q()
 BEGIN
@@ -24,4 +21,8 @@ BEGIN
 END;
 
 CALL iot.query('ais', 'ais02q');
+CALL iot.pause();
+-- We don't set the tumbling, so no tuple will be reused in the following 
window
+CALL iot.heartbeat('ais', 'vessels', 8000);
+CALL iot.resume();
 
diff --git a/sql/backends/monet5/iot/Tests/ais/ais03.sql 
b/sql/backends/monet5/iot/Tests/ais/ais03.sql
--- a/sql/backends/monet5/iot/Tests/ais/ais03.sql
+++ b/sql/backends/monet5/iot/Tests/ais/ais03.sql
@@ -10,18 +10,20 @@ CREATE STREAM TABLE vessels (implicit_ti
 INSERT INTO iot.webserverstreams
SELECT tabl.id, 2 , 8, 's' FROM sys.tables tabl INNER JOIN sys.schemas 
sch ON tabl.schema_id = sch.id WHERE tabl.name = 'vessels' AND sch.name = 'ais';
 
--- We don't set the tumbling, so no tuple will be reused in the following 
window
-CALL iot.heartbeat('ais', 'vessels', 8000); 
-
 --Q3 Currently anchorred ship -- Stream only
 
-CREATE STREAM TABLE ais03r (calc_time timestamp, mmsi int);
+CREATE TABLE ais03r (calc_time timestamp, mmsi int);
 
 CREATE PROCEDURE ais03q()
 BEGIN
INSERT INTO ais03r
-   SELECT current_timestamp, mmsi FROM vessels WHERE nav_status = 
1 AND (implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels GROUP BY mmsi);
+   WITH data_time AS (SELECT current_timestamp AS cur_time)
+   SELECT cur_time, mmsi FROM vessels CROSS JOIN data_time WHERE 
nav_status = 1 AND (implicit_timestamp, mmsi) IN (SELECT 
max(implicit_timestamp), mmsi FROM vessels GROUP BY mmsi);
 END;
 
 CALL iot.query('ais', 'ais03q');
+CALL iot.pause();
+-- We don't set the tumbling, so no tuple will be reused in the following 
window
+CALL iot.heartbeat('ais', 'vessels', 8000);
+CALL iot.resume();
 
diff --git a/sql/backends/monet5/iot/Tests/ais/ais04.sql 
b/sql/backends/monet5/iot/Tests/ais/ais04.sql
--- a/sql/backends/monet5/iot/Tests/ais/ais04.sql
+++ 

MonetDB: iot - Updated some ais tests

[Pedro Ferreira]

Changeset: 98d5fc776077 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=98d5fc776077
Modified Files:
sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
Branch: iot
Log Message:

Updated some ais tests


diffs (43 lines):

diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
@@ -1,4 +1,3 @@
-CREATE SCHEMA ais;
 SET SCHEMA ais;
 SET optimizer = 'iot_pipe';
 
@@ -42,7 +41,7 @@ CREATE PROCEDURE ais07q()
 BEGIN
INSERT INTO ais07r
WITH data AS (SELECT mmsi, geographic_to_cartesian(lat, lon) AS 
calc_point FROM vessels7 WHERE (implicit_timestamp, mmsi) IN (SELECT 
max(implicit_timestamp), mmsi FROM vessels7 WHERE nav_status = 1 GROUP BY 
mmsi)),
-   results AS (SELECT harbor, mmsi FROM data CROSS JOIN 
static_locations, data_time WHERE sys.st_contains(field, calc_point)),
+   results AS (SELECT harbor, mmsi FROM data CROSS JOIN 
static_locations WHERE sys.st_contains(field, calc_point)),
data_time AS (SELECT current_timestamp AS cur_time)
SELECT cur_time, harbor, mmsi FROM results CROSS JOIN data_time;
 END;
diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
@@ -40,7 +40,7 @@ CREATE STREAM TABLE ais10r (calc_time ti
 CREATE PROCEDURE ais10q()
 BEGIN
INSERT INTO ais10r
-   WITH data AS (SELECT harbor, mmsi, sog, sys.Distance(field, 
geographic_to_cartesian(lat, lon)) AS distance FROM vessels10 CROSS JOIN 
static_locations WHERE (implicit_timestamp, mmsi) IN (SELECT 
max(implicit_timestamp), mmsi FROM vessels10 GROUP BY mmsi)),
+   WITH data AS (SELECT harbor, mmsi, sog, sys.st_distance(field, 
geographic_to_cartesian(lat, lon)) AS distance FROM vessels10 CROSS JOIN 
static_locations WHERE (implicit_timestamp, mmsi) IN (SELECT 
max(implicit_timestamp), mmsi FROM vessels10 GROUP BY mmsi)),
data_time AS (SELECT current_timestamp AS cur_time)
SELECT cur_time, harbor, mmsi, distance / sog * 1.852 FROM data 
CROSS JOIN data_time WHERE distance > 0;
 END;
diff --git a/sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql 
b/sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
--- a/sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
+++ b/sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
@@ -37,8 +37,10 @@ CALL iot.pause();
 DELETE FROM iot.webserverstreams;
 DROP PROCEDURE ais11q;
 DROP TABLE vessels11;
+DROP VIEW ais11v;
 DROP TABLE ais11r;
 
 CALL iot.stop();
+SET SCHEMA sys;
 DROP SCHEMA ais;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Added static data to ais test case, fixed some tests

[Pedro Ferreira]

Changeset: cb74e363dffb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cb74e363dffb
Modified Files:
sql/backends/monet5/iot/Tests/ais.sql
sql/backends/monet5/iot/Tests/logger.sql
Branch: iot
Log Message:

Added static data to ais test case, fixed some tests


diffs (283 lines):

diff --git a/sql/backends/monet5/iot/Tests/ais.sql 
b/sql/backends/monet5/iot/Tests/ais.sql
--- a/sql/backends/monet5/iot/Tests/ais.sql
+++ b/sql/backends/monet5/iot/Tests/ais.sql
@@ -2,8 +2,8 @@ CREATE SCHEMA ais;
 SET SCHEMA ais;
 SET optimizer = 'iot_pipe';
 
-/* calculate distance in kms between two coordinates 
http://www.movable-type.co.uk/scripts/latlong.html
-   so we don't need to create a geometry element when working only with stream 
data (the iot web server doesn't support geom types yet) */
+-- Calculate distance in kms between two coordinates: 
http://www.movable-type.co.uk/scripts/latlong.html
+-- Therefore we don't need to create a geometry element when working only with 
stream data (the iot web server doesn't support geom types yet)
 CREATE FUNCTION km_distance(lat1 FLOAT, lon1 FLOAT, lat2 FLOAT, lon2 FLOAT) 
RETURNS FLOAT
 BEGIN
DECLARE deg_to_rad FLOAT, deg_to_rad_div FLOAT, aux FLOAT;
@@ -13,8 +13,8 @@ BEGIN
RETURN 12742 * sys.atan(sys.sqrt(aux), sys.sqrt(1 - aux));
 END;
 
--- returns a geometry point from latitude and longitude 
https://rbrundritt.wordpress.com/2008/10/14/conversion-between-spherical-and-cartesian-coordinates-systems/
-CREATE FUNCTION geographical_to_cartesian(lat FLOAT, lon FLOAT) RETURNS 
Geometry
+-- Returns a geometry point from latitude and longitude: 
https://rbrundritt.wordpress.com/2008/10/14/conversion-between-spherical-and-cartesian-coordinates-systems/
+CREATE FUNCTION geographic_to_cartesian(lat FLOAT, lon FLOAT) RETURNS POINT
 BEGIN
DECLARE deg_to_rad FLOAT, lat_rad FLOAT, lon_rad FLOAT, aux1 FLOAT, 
aux2 FLOAT;
SET deg_to_rad = pi() / 180;
@@ -25,43 +25,80 @@ BEGIN
RETURN sys.st_makepoint(aux2 * sys.cos(lon_rad), aux2 * 
sys.sin(lon_rad), 6371 * sys.sin(lat_rad));
 END;
 
-CREATE TABLE static_locations (loc Geometry); /* TODO Populate this table */ 
+CREATE TABLE static_locations (harbor CHAR(32), field POLYGON); /* All major 
harbors of Amsterdam: https://en.wikipedia.org/wiki/Port_of_Amsterdam#Geography 
*/
 
-CREATE STREAM TABLE vessels (implicit_timestamp timestamp, mmsi int, lat real, 
lon real, nav_status tinyint, sog real, rotais smallint);
-CREATE STREAM TABLE stations (implicit_timestamp timestamp, mmsi int, lat 
real, lon real, pos_dev tinyint);
+INSERT INTO static_locations VALUES ('Amerikahaven', 'POLYGON( (3871.44456893 
322.23523795 5049.52694825, 3873.75021533 323.064093023 5047.70540553, 
3873.71867646 323.885373981 5047.67697859, 3873.5017397 324.872292495 
5047.78003345, 3872.7938342 325.19853123 5048.30217331, 3872.60164177 
324.785093693 5048.4769, 3872.6741224 323.809744716 5048.48327629, 
3871.54981295 323.35946261 5049.37438737) )'); /* Amerikahaven */
+INSERT INTO static_locations VALUES ('Petroleumhaven', 'POLYGON( 
(3871.97792572 328.849674441 5048.69149725, 3872.33795265 328.725455306 
5048.42345247, 3872.43401269 329.066629268 5048.32754195, 3872.11428671 
329.28613 5048.55835959, 3872.00301454 329.044638923 5048.65955289) )'); /* 
Petroleumhaven */
+INSERT INTO static_locations VALUES ('Oostelijkhaven', 'POLYGON( 
(3874.76945837 332.896017406 5046.28406712, 3874.99350047 332.95621223 
5046.1080579, 3875.1279382 334.058396133 5045.93197046, 3875.5579763 
334.604372405 5045.56550708, 3875.35055079 335.984299981 5045.63312763, 
3874.41663614 335.616780364 5046.3747489, 3874.6428613 333.721315622 
5046.32676121) )'); /* Oostelijkhaven */
+INSERT INTO static_locations VALUES ('Afrikahaven', 'POLYGON( (3871.30476187 
320.817833925 5049.72438438, 3872.88902009 321.220747092 5048.4838189, 
3872.96603 321.966084418 5048.37726116, 3872.74497591 322.035368331 
5048.54242065, 3871.38317977 321.685638604 5049.60905668) )'); /* Afrikahaven */
+INSERT INTO static_locations VALUES ('DeRuijterkadehaven', 'POLYGON( 
(3874.84907684 331.832795545 5046.29295895, 3874.5621738 332.100597943 
5046.49563105, 3874.74221319 332.592560967 5046.32499644, 3875.14500709 
332.378589729 5046.02979055) )'); /* DeRuijterkadehaven */
+INSERT INTO static_locations VALUES ('Coenhaven', 'POLYGON( (3872.18320026 
329.539035698 5048.48910938, 3872.7184903 329.096658095 5048.10735667, 
3872.88840793 329.5493504 5047.9474646, 3872.80986545 329.954639206 
5047.98124819, 3872.54376258 330.007944404 5048.18190674, 3872.3152549 
329.754769754 5048.37373404) )'); /* Coenhaven */
+INSERT INTO static_locations VALUES ('Houthaven', 'POLYGON( (3872.87117926 
330.512107085 5047.89773826, 3872.96624169 330.37417078 5047.83383225, 
3873.54601208 330.634050719 5047.37193169, 3873.94528714 331.083171623 
5047.03604561, 3873.97858621 331.533215981 5046.98094313, 3873.54647189 
331.388091275 5047.32212793, 3872.86905716 

MonetDB: iot - Splited AIS tests, fixed some MAL error messages ...

[Pedro Ferreira]

Changeset: 72bb2608f2f5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=72bb2608f2f5
Added Files:
sql/backends/monet5/iot/Tests/ais/ais01.sql
sql/backends/monet5/iot/Tests/ais/ais02.sql
sql/backends/monet5/iot/Tests/ais/ais03.sql
sql/backends/monet5/iot/Tests/ais/ais04.sql
sql/backends/monet5/iot/Tests/ais/ais05.sql
sql/backends/monet5/iot/Tests/ais/ais06.sql
sql/backends/monet5/iot/Tests/ais/ais07.sql
sql/backends/monet5/iot/Tests/ais/ais08.sql
sql/backends/monet5/iot/Tests/ais/ais09.sql
sql/backends/monet5/iot/Tests/ais/ais10.sql
sql/backends/monet5/iot/Tests/ais/ais11.sql
Removed Files:
sql/backends/monet5/iot/Tests/ais.sql
Modified Files:
sql/backends/monet5/iot/Tests/iot13.sql
sql/backends/monet5/iot/petrinet.c
Branch: iot
Log Message:

Splited AIS tests, fixed some MAL error messages on Petrinet


diffs (truncated from 926 to 300 lines):

diff --git a/sql/backends/monet5/iot/Tests/ais.sql 
b/sql/backends/monet5/iot/Tests/ais.sql
deleted file mode 100644
--- a/sql/backends/monet5/iot/Tests/ais.sql
+++ /dev/null
@@ -1,238 +0,0 @@
-CREATE SCHEMA ais;
-SET SCHEMA ais;
-SET optimizer = 'iot_pipe';
-
--- Calculate distance in kms between two coordinates: 
http://www.movable-type.co.uk/scripts/latlong.html
--- Therefore we don't need to create a geometry element when working only with 
stream data (the iot web server doesn't support geom types yet)
-CREATE FUNCTION km_distance(lat1 FLOAT, lon1 FLOAT, lat2 FLOAT, lon2 FLOAT) 
RETURNS FLOAT
-BEGIN
-   DECLARE deg_to_rad FLOAT, deg_to_rad_div FLOAT, aux FLOAT;
-   SET deg_to_rad = pi() / 180;
-   SET deg_to_rad_div = deg_to_rad / 2;
-   SET aux = sys.power(sys.sin((lat2 - lat1) * deg_to_rad_div), 2) + 
sys.cos(lat1 * deg_to_rad) * sys.cos(lat2 * deg_to_rad) * 
sys.power(sys.sin((lon2 - lon1) * deg_to_rad_div), 2);
-   RETURN 12742 * sys.atan(sys.sqrt(aux), sys.sqrt(1 - aux));
-END;
-
--- Returns a geometry point from latitude and longitude: 
https://rbrundritt.wordpress.com/2008/10/14/conversion-between-spherical-and-cartesian-coordinates-systems/
-CREATE FUNCTION geographic_to_cartesian(lat FLOAT, lon FLOAT) RETURNS POINT
-BEGIN
-   DECLARE deg_to_rad FLOAT, lat_rad FLOAT, lon_rad FLOAT, aux1 FLOAT, 
aux2 FLOAT;
-   SET deg_to_rad = pi() / 180;
-   SET lat_rad = lat * deg_to_rad;
-   SET lon_rad = lon * deg_to_rad;
-   SET aux1 = sys.cos(lat_rad);
-   SET aux2 = 6371 * aux1;
-   RETURN sys.st_makepoint(aux2 * sys.cos(lon_rad), aux2 * 
sys.sin(lon_rad), 6371 * sys.sin(lat_rad));
-END;
-
-CREATE TABLE static_locations (harbor CHAR(32), field POLYGON); /* All major 
harbors of Amsterdam: https://en.wikipedia.org/wiki/Port_of_Amsterdam#Geography 
*/
-
-INSERT INTO static_locations VALUES ('Amerikahaven', 'POLYGON( (3871.44456893 
322.23523795 5049.52694825, 3873.75021533 323.064093023 5047.70540553, 
3873.71867646 323.885373981 5047.67697859, 3873.5017397 324.872292495 
5047.78003345, 3872.7938342 325.19853123 5048.30217331, 3872.60164177 
324.785093693 5048.4769, 3872.6741224 323.809744716 5048.48327629, 
3871.54981295 323.35946261 5049.37438737) )'); /* Amerikahaven */
-INSERT INTO static_locations VALUES ('Petroleumhaven', 'POLYGON( 
(3871.97792572 328.849674441 5048.69149725, 3872.33795265 328.725455306 
5048.42345247, 3872.43401269 329.066629268 5048.32754195, 3872.11428671 
329.28613 5048.55835959, 3872.00301454 329.044638923 5048.65955289) )'); /* 
Petroleumhaven */
-INSERT INTO static_locations VALUES ('Oostelijkhaven', 'POLYGON( 
(3874.76945837 332.896017406 5046.28406712, 3874.99350047 332.95621223 
5046.1080579, 3875.1279382 334.058396133 5045.93197046, 3875.5579763 
334.604372405 5045.56550708, 3875.35055079 335.984299981 5045.63312763, 
3874.41663614 335.616780364 5046.3747489, 3874.6428613 333.721315622 
5046.32676121) )'); /* Oostelijkhaven */
-INSERT INTO static_locations VALUES ('Afrikahaven', 'POLYGON( (3871.30476187 
320.817833925 5049.72438438, 3872.88902009 321.220747092 5048.4838189, 
3872.96603 321.966084418 5048.37726116, 3872.74497591 322.035368331 
5048.54242065, 3871.38317977 321.685638604 5049.60905668) )'); /* Afrikahaven */
-INSERT INTO static_locations VALUES ('DeRuijterkadehaven', 'POLYGON( 
(3874.84907684 331.832795545 5046.29295895, 3874.5621738 332.100597943 
5046.49563105, 3874.74221319 332.592560967 5046.32499644, 3875.14500709 
332.378589729 5046.02979055) )'); /* DeRuijterkadehaven */
-INSERT INTO static_locations VALUES ('Coenhaven', 'POLYGON( (3872.18320026 
329.539035698 5048.48910938, 3872.7184903 329.096658095 5048.10735667, 
3872.88840793 329.5493504 5047.9474646, 3872.80986545 329.954639206 
5047.98124819, 3872.54376258 330.007944404 5048.18190674, 3872.3152549 
329.754769754 5048.37373404) )'); /* Coenhaven */
-INSERT INTO static_locations VALUES ('Houthaven', 'POLYGON( (3872.87117926 
330.512107085 5047.89773826, 3872.96624169 

MonetDB: iot - Added first version of AIS use case to test the s...

[Pedro Ferreira]

Changeset: 8729da329f5c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8729da329f5c
Added Files:
sql/backends/monet5/iot/Tests/ais.sql
Branch: iot
Log Message:

Added first version of AIS use case to test the streaming engine


diffs (206 lines):

diff --git a/sql/backends/monet5/iot/Tests/ais.sql 
b/sql/backends/monet5/iot/Tests/ais.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/iot/Tests/ais.sql
@@ -0,0 +1,201 @@
+CREATE SCHEMA ais;
+SET SCHEMA ais;
+SET optimizer = 'iot_pipe';
+
+/* calculate distance in kms between two coordinates 
http://www.movable-type.co.uk/scripts/latlong.html
+   so we don't need to create a geometry element when working only with stream 
data (the iot web server doesn't support geom types yet) */
+CREATE FUNCTION km_distance(lat1 FLOAT, lon1 FLOAT, lat2 FLOAT, lon2 FLOAT) 
RETURNS FLOAT
+BEGIN
+   DECLARE deg_to_rad FLOAT, deg_to_rad_div FLOAT, aux FLOAT;
+   SET deg_to_rad = pi() / 180;
+   SET deg_to_rad_div = deg_to_rad / 2;
+   SET aux = sys.power(sys.sin((lat2 - lat1) * deg_to_rad_div), 2) + 
sys.cos(lat1 * deg_to_rad) * sys.cos(lat2 * deg_to_rad) * 
sys.power(sys.sin((lon2 - lon1) * deg_to_rad_div), 2);
+   RETURN 12742 * sys.atan(sys.sqrt(aux), sys.sqrt(1 - aux));
+END;
+
+-- returns a geometry point from latitude and longitude 
https://rbrundritt.wordpress.com/2008/10/14/conversion-between-spherical-and-cartesian-coordinates-systems/
+CREATE FUNCTION geographical_to_cartesian(lat FLOAT, lon FLOAT) RETURNS 
Geometry
+BEGIN
+   DECLARE deg_to_rad FLOAT, lat_rad FLOAT, lon_rad FLOAT, aux1 FLOAT, 
aux2 FLOAT;
+   SET deg_to_rad = pi() / 180;
+   SET lat_rad = lat * deg_to_rad;
+   SET lon_rad = lon * deg_to_rad;
+   SET aux1 = sys.cos(lat_rad);
+   SET aux2 = 6371 * aux1;
+   RETURN sys.st_makepoint(aux2 * sys.cos(lon_rad), aux2 * 
sys.sin(lon_rad), 6371 * sys.sin(lat_rad));
+END;
+
+CREATE TABLE static_locations (loc Geometry); /* TODO Populate this table */ 
+
+CREATE STREAM TABLE vessels (implicit_timestamp timestamp, mmsi int, lat real, 
lon real, nav_status tinyint, sog real, rotais smallint);
+CREATE STREAM TABLE stations (implicit_timestamp timestamp, mmsi int, lat 
real, lon real, pos_dev tinyint);
+
+CREATE STREAM TABLE vessels1 (LIKE vessels);
+CREATE STREAM TABLE vessels2 (LIKE vessels);
+CREATE STREAM TABLE vessels3 (LIKE vessels);
+CREATE STREAM TABLE vessels4 (LIKE vessels);
+CREATE STREAM TABLE vessels5 (LIKE vessels);
+CREATE STREAM TABLE vessels6 (LIKE vessels);
+CREATE STREAM TABLE vessels7 (LIKE vessels);
+CREATE STREAM TABLE vessels8 (LIKE vessels);
+CREATE STREAM TABLE vessels9 (LIKE vessels);
+CREATE STREAM TABLE vessels10 (LIKE vessels);
+CREATE STREAM TABLE vessels11 (LIKE vessels);
+
+CALL iot.heartbeat('ais','vessels',8000); /*Position reports are sent every 
3-5 seconds so we can run the query for the tuples arrived in the last */
+
+CREATE PROCEDURE ais00q() /*Provide data for each query*/
+BEGIN
+   INSERT INTO vessels1 SELECT * FROM vessels;
+   INSERT INTO vessels2 SELECT * FROM vessels;
+   INSERT INTO vessels3 SELECT * FROM vessels;
+   INSERT INTO vessels4 SELECT * FROM vessels;
+   INSERT INTO vessels5 SELECT * FROM vessels;
+   INSERT INTO vessels6 SELECT * FROM vessels;
+   INSERT INTO vessels7 SELECT * FROM vessels;
+   INSERT INTO vessels8 SELECT * FROM vessels;
+   INSERT INTO vessels9 SELECT * FROM vessels;
+   INSERT INTO vessels10 SELECT * FROM vessels;
+   INSERT INTO vessels11 SELECT * FROM vessels;
+END;
+
+--Q1 Calculate speed of ships per hour (in knots) -- Stream only
+
+CREATE TABLE ais01r (calc_time timestamp, mmsi int, sog real);
+
+CREATE PROCEDURE ais01q()
+BEGIN
+   INSERT INTO ais01r
+   SELECT current_timestamp, mmsi, sog FROM vessels1 WHERE 
(implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels1 GROUP BY mmsi);
+END;
+
+--Q2 Number of ship per hour -- Stream only
+
+CREATE TABLE ais02r (calc_time timestamp, number_ships int);
+
+CREATE PROCEDURE ais02q()
+BEGIN
+   INSERT INTO ais02r
+   SELECT current_timestamp, count(DISTINCT mmsi) FROM vessels2;
+END;
+
+--Q3 Currently anchorred ship -- Stream only
+
+CREATE TABLE ais03r (calc_time timestamp, mmsi int);
+
+CREATE PROCEDURE ais03q()
+BEGIN
+   INSERT INTO ais03r
+   SELECT current_timestamp, mmsi FROM vessels3 WHERE nav_status = 
1 AND (implicit_timestamp, mmsi) IN (SELECT max(implicit_timestamp), mmsi FROM 
vessels3 GROUP BY mmsi);
+END;
+
+--Q4 Ship turning degree > 180 -- Stream only
+
+CREATE TABLE ais04r (calc_time timestamp, mmsi int);
+
+CREATE PROCEDURE ais04q()
+BEGIN
+   INSERT INTO ais04r
+   SELECT current_timestamp, mmsi FROM vessels4 WHERE 
sys.abs(rotais) > 180 AND (implicit_timestamp, mmsi) IN (SELECT 
max(implicit_timestamp), mmsi FROM vessels4 GROUP BY mmsi);
+END;
+
+--Q5 Closest ship to each other -- Stream only
+

MonetDB: iot - Bug found on tuple based windows

[Pedro Ferreira]

Changeset: 352699331b28 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=352699331b28
Added Files:
sql/backends/monet5/iot/Tests/bug05.sql
Modified Files:
clients/iotapi/documentation/websockets_api.rst
clients/iotapi/src/WebSockets/jsonschemas.py
sql/backends/monet5/iot/Tests/All
Branch: iot
Log Message:

Bug found on tuple based windows


diffs (90 lines):

diff --git a/clients/iotapi/documentation/websockets_api.rst 
b/clients/iotapi/documentation/websockets_api.rst
--- a/clients/iotapi/documentation/websockets_api.rst
+++ b/clients/iotapi/documentation/websockets_api.rst
@@ -15,28 +15,28 @@ The client must always provide a JSON st
 
 The following sections explain the available :code:`request` fields.
 
-sub/subscribe
--
+subscribe
+-
 
 Subscribes for new basket creations from a specific stream. Whenever a basket 
is created, the server sends a notification message indicating the number of 
inserted tuples in the new basket. The user has to specify the stream's name 
and schema. To subscribe to a temperature stream, the following would suffice:
 
 .. code-block:: json
 
 {
-"request": "sub",
+"request": "subscribe",
 "schema": "measures",
 "stream": "temperature"
 }
 
-unsub/unsubscribe
--
+unsubscribe
+---
 
 Unsubscribes a previous subscribed stream for a client. The user has to 
specify the stream's name and schema. The example is the same as above, just 
changing the request keyword.
 
 .. code-block:: json
 
 {
-"request": "unsub",
+"request": "unsubscribe",
 "schema": "measures",
 "stream": "temperature"
 }
diff --git a/clients/iotapi/src/WebSockets/jsonschemas.py 
b/clients/iotapi/src/WebSockets/jsonschemas.py
--- a/clients/iotapi/src/WebSockets/jsonschemas.py
+++ b/clients/iotapi/src/WebSockets/jsonschemas.py
@@ -1,5 +1,5 @@
-SUBSCRIBE_OPTS = ["sub", "subscribe"]
-UNSUBSCRIBE_OPTS = ["unsub", "unsubscribe"]
+SUBSCRIBE_OPTS = ["subscribe"]
+UNSUBSCRIBE_OPTS = ["unsubscribe"]
 INFO_OPTS = ["info"]
 READ_OPTS = ["read"]
 
diff --git a/sql/backends/monet5/iot/Tests/All 
b/sql/backends/monet5/iot/Tests/All
--- a/sql/backends/monet5/iot/Tests/All
+++ b/sql/backends/monet5/iot/Tests/All
@@ -2,6 +2,7 @@ bug01
 bug02
 bug03
 bug04
+bug05
 iot00
 iot02
 iot03
diff --git a/sql/backends/monet5/iot/Tests/bug05.sql 
b/sql/backends/monet5/iot/Tests/bug05.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/iot/Tests/bug05.sql
@@ -0,0 +1,25 @@
+SET optimizer = 'iot_pipe';
+
+CREATE STREAM TABLE testing (a timestamp, b int, c real);
+CALL iot.window('sys', 'testing', 3);
+CREATE TABLE testout (a timestamp, b int, c real);
+
+CREATE PROCEDURE cquery()
+BEGIN  
+   INSERT INTO testout SELECT * FROM testing;
+END;
+
+CALL iot.query('sys', 'cquery');
+CALL iot.resume();
+
+INSERT INTO testing VALUES (now(), 1, 1);
+INSERT INTO testing VALUES (now(), 2, 2);
+INSERT INTO testing VALUES (now(), 3, 3);
+
+CALL iot.show('sys', 'cquery');
+CALL iot.stop();
+
+DROP PROCEDURE cquery;
+DROP TABLE testout;
+DROP TABLE testing;
+
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Updated requirements.txt file, removed mapi flush...

[Pedro Ferreira]

Changeset: f8a016c9057b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8a016c9057b
Modified Files:
clients/iotclient/requirements.txt
clients/iotclient/src/Streams/streams.py
Branch: iot
Log Message:

Updated requirements.txt file, removed mapi flush baskets call comment.


diffs (30 lines):

diff --git a/clients/iotclient/requirements.txt 
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,4 +1,4 @@
-fake-factory>=0.5.7
+fake-factory>=0.5.8
 Flask-RESTful>=0.3.5
 IPy>=0.83
 jsonschema>=2.5.1
diff --git a/clients/iotclient/src/Streams/streams.py 
b/clients/iotclient/src/Streams/streams.py
--- a/clients/iotclient/src/Streams/streams.py
+++ b/clients/iotclient/src/Streams/streams.py
@@ -64,7 +64,7 @@ class BaseIOTStream:
 if dirs:
 for elem in dirs:  # for each directory found, flush it
 dir_path = os.path.join(self._base_path, str(elem))
-# mapi_flush_baskets(self._connection, self._schema_name, 
self._stream_name, dir_path)
+mapi_flush_baskets(self._connection, self._schema_name, 
self._stream_name, dir_path)
 self._baskets_counter = max(dirs) + 1  # increment current 
basket number
 else:
 self._baskets_counter = 1
@@ -138,7 +138,7 @@ class BaseIOTStream:
 
 def flush_baskets(self, last=False):  # the monitor has to be acquired in 
write mode before running this method!!!
 # write the tuple count in the basket
-# mapi_flush_baskets(self._connection, self._schema_name, 
self._stream_name, self._current_base_path)
+mapi_flush_baskets(self._connection, self._schema_name, 
self._stream_name, self._current_base_path)
 
 if not last:  # when stopping the stream, we don't want to continue to 
create more baskets files
 self._tuples_in_per_basket = 0
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Updated basket import procedure statement

[Pedro Ferreira]

Changeset: c92046756c56 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c92046756c56
Modified Files:
clients/iotclient/src/Settings/mapiconnection.py
Branch: iot
Log Message:

Updated basket import procedure statement


diffs (12 lines):

diff --git a/clients/iotclient/src/Settings/mapiconnection.py 
b/clients/iotclient/src/Settings/mapiconnection.py
--- a/clients/iotclient/src/Settings/mapiconnection.py
+++ b/clients/iotclient/src/Settings/mapiconnection.py
@@ -104,7 +104,7 @@ def mapi_delete_stream(connection, conca
 
 def mapi_flush_baskets(connection, schema, stream, baskets):
 try:
-connection.execute(''.join(["CALL iot.basket('", schema, "','", 
stream, "','", baskets, "')"]))
+connection.execute(''.join(["CALL iot.import('", schema, "','", 
stream, "','", baskets, "')"]))
 connection.commit()
 except BaseException as ex:
 add_log(40, ex)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Changed the jar file name to avoid conf...

[Pedro Ferreira]

Changeset: 595d455b70a2 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=595d455b70a2
Modified Files:
build.xml
tests/build.xml
Branch: embedded
Log Message:

Changed the jar file name to avoid conflicts with the original one.


diffs (24 lines):

diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -32,7 +32,7 @@ Copyright 1997 - July 2008 CWI, August 2
 
   
 
-  
+  
   
   
   
diff --git a/tests/build.xml b/tests/build.xml
--- a/tests/build.xml
+++ b/tests/build.xml
@@ -25,7 +25,7 @@ Copyright 1997 - July 2008 CWI, August 2
   
   
   
+value="${jardir}/monetdb-jdbc-new-${JDBC_MAJOR}.${JDBC_MINOR}.jar" />
   
   https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Fixed SQLException generation.

[Pedro Ferreira]

Changeset: 0cfaeec9afcb for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=0cfaeec9afcb
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
Branch: embedded
Log Message:

Fixed SQLException generation.


diffs (119 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -1,16 +1,16 @@
 package nl.cwi.monetdb.jdbc;
 
-import nl.cwi.monetdb.mcl.connection.*;
+import nl.cwi.monetdb.mcl.connection.ControlCommands;
+import nl.cwi.monetdb.mcl.connection.IMonetDBLanguage;
+import nl.cwi.monetdb.mcl.connection.MCLException;
 import nl.cwi.monetdb.mcl.connection.SenderThread;
+import nl.cwi.monetdb.mcl.protocol.AbstractProtocol;
 import nl.cwi.monetdb.mcl.protocol.ProtocolException;
-import nl.cwi.monetdb.mcl.protocol.AbstractProtocol;
 import nl.cwi.monetdb.mcl.protocol.ServerResponses;
 import nl.cwi.monetdb.mcl.protocol.StarterHeaders;
 import nl.cwi.monetdb.mcl.responses.*;
-import nl.cwi.monetdb.mcl.responses.DataBlockResponse;
-import nl.cwi.monetdb.mcl.responses.ResultSetResponse;
 
-import java.io.*;
+import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.sql.*;
 import java.util.*;
@@ -80,7 +80,7 @@ public abstract class MonetConnection ex
 // See javadoc for documentation about WeakHashMap if you don't know what 
it does !!!NOW!!!
 // (only when you deal with it of course)
 /** A Map containing all (active) Statements created from this Connection 
*/
-private Map statements = new WeakHashMap();
+private Map statements = new WeakHashMap<>();
 /** The number of results we receive from the server at once */
 private int curReplySize = -1; // the server by default uses -1 (all)
 /** Whether or not BLOB is mapped to LONGVARBINARY within the driver */
@@ -1033,12 +1033,12 @@ public abstract class MonetConnection ex
 if (rs != null) {
 try {
 rs.close();
-} catch (Exception e2) {}
+} catch (Exception ignored) {}
 }
 if (stmt != null) {
 try {
 stmt.close();
-} catch (Exception e2) {}
+} catch (Exception ignored) {}
 }
 }
 return false;
@@ -1192,11 +1192,8 @@ public abstract class MonetConnection ex
 if (schema == null)
 throw new SQLException("Missing schema name", "M1M05");
 
-Statement st = createStatement();
-try {
+try (Statement st = createStatement()) {
 st.execute("SET SCHEMA \"" + schema + "\"");
-} finally {
-st.close();
 }
 }
 
@@ -1318,7 +1315,7 @@ public abstract class MonetConnection ex
  * @param command the exact string to send to MonetDB
  * @throws SQLException if an IO exception or a database error occurs
  */
-public void sendIndependentCommand(String command) throws SQLException {
+void sendIndependentCommand(String command) throws SQLException {
 try {
 protocol.writeNextQuery(language.getQueryTemplateIndex(0), 
command, language.getQueryTemplateIndex(1));
 protocol.waitUntilPrompt();
@@ -1341,7 +1338,7 @@ public abstract class MonetConnection ex
  *
  * @param reason the warning message
  */
-public void addWarning(String reason, String sqlstate) {
+protected void addWarning(String reason, String sqlstate) {
 if (warnings == null) {
 warnings = new SQLWarning(reason, sqlstate);
 } else {
@@ -1381,7 +1378,7 @@ public abstract class MonetConnection ex
  * @param rstype the type of result sets to produce
  * @param rsconcur the concurrency of result sets to produce
  */
-public ResponseList(int cachesize, int maxrows, int rstype, int 
rsconcur) {
+ResponseList(int cachesize, int maxrows, int rstype, int rsconcur) {
 this.cachesize = cachesize;
 this.maxrows = maxrows;
 this.rstype = rstype;
@@ -1507,7 +1504,7 @@ public abstract class MonetConnection ex
 protocol.waitUntilPrompt();
 
 // {{{ set reply size
-/**
+/*
  * Change the reply size of the server.  If the given value is 
the same as the current value known
  * to use, then ignore this call.  If it is set to 0 we get a 
prompt after the server sent it's
  * header.
@@ -1677,14 +1674,14 @@ public abstract class MonetConnection ex
 }
 if (error != null) {
 SQLException ret = null;
-String[] errors = error.split("\n");
-for 

monetdb-java: embedded - Small bug fixes

[Pedro Ferreira]

Changeset: 2867df9b4d2e for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=2867df9b4d2e
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java
src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java
Branch: embedded
Log Message:

Small bug fixes


diffs (108 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -1514,7 +1514,7 @@ public abstract class MonetConnection ex
 int size = (cachesize != 0 && !isEmbedded) ? cachesize : 
MonetConnection.this.getDefFetchsize();
 size = maxrows != 0 ? Math.min(maxrows, size) : size;
 // don't do work if it's not needed
-if (!language.getRepresentation().equals("sql") && size != 
curReplySize &&
+if (!isEmbedded && language.getRepresentation().equals("sql") 
&& size != curReplySize &&
 !Arrays.deepEquals(templ, 
language.getCommandTemplates())) {
 sendControlCommand(ControlCommands.REPLY_SIZE, size);
 // store the reply size after a successful change
@@ -1557,7 +1557,7 @@ public abstract class MonetConnection ex
 case StarterHeaders.Q_TABLE:
 case StarterHeaders.Q_PREPARE: {
 res = 
protocol.getNextResultSetResponse(MonetConnection.this,
-ResponseList.this, this.seqnr);
+ResponseList.this, this.seqnr, 
this.maxrows);
 ResultSetResponse rsreponse = 
(ResultSetResponse) res;
 // only add this resultset to the 
hashmap if it can possibly
 // have an additional datablock
diff --git a/src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java 
b/src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java
--- a/src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java
@@ -9,11 +9,7 @@
 package nl.cwi.monetdb.mcl.protocol;
 
 import nl.cwi.monetdb.jdbc.MonetConnection;
-import nl.cwi.monetdb.mcl.responses.AutoCommitResponse;
-import nl.cwi.monetdb.mcl.responses.SchemaResponse;
-import nl.cwi.monetdb.mcl.responses.UpdateResponse;
-import nl.cwi.monetdb.mcl.responses.DataBlockResponse;
-import nl.cwi.monetdb.mcl.responses.ResultSetResponse;
+import nl.cwi.monetdb.mcl.responses.*;
 
 import java.io.IOException;
 import java.text.ParsePosition;
@@ -177,11 +173,12 @@ public abstract class AbstractProtocol {
  * @param con The current MonetDB's JDBC connection
  * @param list The Response List this result set will belong to
  * @param seqnr The sequence number of this result set on the Response List
+ * @param maxrows A maxrows to set if so
  * @return The ResultSet instance
  * @throws ProtocolException If an error in the underlying connection 
happened.
  */
 public abstract ResultSetResponse getNextResultSetResponse(MonetConnection 
con, MonetConnection.ResponseList list,
-   int seqnr) 
throws ProtocolException;
+   int seqnr, int 
maxrows) throws ProtocolException;
 
 /**
  * Gets the next UpdateResponse response from the server.
diff --git 
a/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java 
b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java
--- a/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java
@@ -10,14 +10,14 @@ package nl.cwi.monetdb.mcl.protocol.oldm
 
 import nl.cwi.monetdb.jdbc.MonetConnection;
 import nl.cwi.monetdb.mcl.connection.mapi.OldMapiSocket;
+import nl.cwi.monetdb.mcl.protocol.AbstractProtocol;
 import nl.cwi.monetdb.mcl.protocol.ProtocolException;
-import nl.cwi.monetdb.mcl.protocol.AbstractProtocol;
 import nl.cwi.monetdb.mcl.protocol.ServerResponses;
 import nl.cwi.monetdb.mcl.protocol.StarterHeaders;
 import nl.cwi.monetdb.mcl.responses.AutoCommitResponse;
-import nl.cwi.monetdb.mcl.responses.UpdateResponse;
 import nl.cwi.monetdb.mcl.responses.DataBlockResponse;
 import nl.cwi.monetdb.mcl.responses.ResultSetResponse;
+import nl.cwi.monetdb.mcl.responses.UpdateResponse;
 
 import java.io.IOException;
 import java.nio.CharBuffer;
@@ -121,9 +121,11 @@ public class OldMapiProtocol extends Abs
 this.currentServerResponseHeader = 

monetdb-java: embedded - Small fixes for the embedded connection.

[Pedro Ferreira]

Changeset: 8af0a7387b4e for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=8af0a7387b4e
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetDataSource.java
Branch: embedded
Log Message:

Small fixes for the embedded connection.


diffs (102 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -2,7 +2,6 @@ package nl.cwi.monetdb.jdbc;
 
 import nl.cwi.monetdb.mcl.connection.*;
 import nl.cwi.monetdb.mcl.connection.SenderThread;
-import nl.cwi.monetdb.mcl.connection.mapi.MapiLanguage;
 import nl.cwi.monetdb.mcl.protocol.ProtocolException;
 import nl.cwi.monetdb.mcl.protocol.AbstractProtocol;
 import nl.cwi.monetdb.mcl.protocol.ServerResponses;
@@ -394,7 +393,7 @@ public abstract class MonetConnection ex
  */
 @Override
 public DatabaseMetaData getMetaData() throws SQLException {
-if (this.language != MapiLanguage.LANG_SQL) {
+if (!this.language.getRepresentation().equals("sql")) {
 throw new SQLException("This method is only supported in SQL 
mode", "M0M04");
 }
 return new MonetDatabaseMetaData(this);
@@ -1511,7 +1510,7 @@ public abstract class MonetConnection ex
 int size = (cachesize != 0 && !isEmbedded) ? cachesize : 
MonetConnection.this.getDefFetchsize();
 size = maxrows != 0 ? Math.min(maxrows, size) : size;
 // don't do work if it's not needed
-if (language == MapiLanguage.LANG_SQL && size != curReplySize 
&&
+if (!language.getRepresentation().equals("sql") && size != 
curReplySize &&
 !Arrays.deepEquals(templ, 
language.getCommandTemplates())) {
 sendControlCommand(ControlCommands.REPLY_SIZE, size);
 // store the reply size after a successful change
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetDataSource.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetDataSource.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetDataSource.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetDataSource.java
@@ -27,7 +27,7 @@ import java.util.logging.Logger;
  *
  * Additionally, pooled connections can be used when using a DataSource.
  *
- * @author Fabian Groffen
+ * @author Fabian Groffen, Pedro Ferreira
  * @version 0.1
  */
 public class MonetDataSource extends MonetWrapper implements DataSource {
@@ -37,7 +37,7 @@ public class MonetDataSource extends Mon
 private String description = "MonetDB database";
private String url = "jdbc:monetdb://localhost/";
 private int loginTimeout;
-private boolean isEmbedded;
+private String directory;
 private final MonetDriver driver = new MonetDriver();
 
// the following properties are also standard:
@@ -75,8 +75,9 @@ public class MonetDataSource extends Mon
 if (loginTimeout > 0) {
 props.put("so_timeout", Integer.toString(loginTimeout));
 }
-   if(isEmbedded) {
+   if(directory != null) {
 props.put("embedded", "true");
+   props.put("directory", directory);
 }
return driver.connect(url, props);
}
@@ -184,21 +185,30 @@ public class MonetDataSource extends Mon
}
 
 /**
+ * Gets the directory value
+ *
+ * @return the directory value
+ */
+public String getDirectory() {
+return directory;
+}
+
+/**
+ * Sets the directory value, meaning it wil start an embedded connection
+ *
+ * @param directory The directory location
+ */
+public void setDirectory(String directory) {
+this.directory = directory;
+}
+
+/**
  * Gets the embedded connection directory. If not, then a MAPI connection 
will be created instead.
  *
  * @return If the connection will be embedded. If not, then a MAPI 
connection will be created instead.
  */
 public boolean isEmbedded() {
-return isEmbedded;
-}
-
-/**
- * Sets the connection to be embedded
- *
- * @param isEmbedded A boolean to indicate if the connection will be 
embedded
- */
-public void setIsEmbedded(boolean isEmbedded) {
-this.isEmbedded = isEmbedded;
+return directory != null;
 }
 
 /**
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - After a lot of suffering, finnaly passe...

[Pedro Ferreira]

Changeset: e026fe73bb5e for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=e026fe73bb5e
Added Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetURL.java
src/main/java/nl/cwi/monetdb/mcl/connection/helpers/TimestampHelper.java
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetINET.java
src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java

src/main/java/nl/cwi/monetdb/mcl/connection/helpers/GregorianCalendarParser.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
src/main/java/nl/cwi/monetdb/mcl/protocol/AbstractProtocol.java
src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java

src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParser.java
src/main/java/nl/cwi/monetdb/mcl/responses/DataBlockResponse.java
src/main/java/nl/cwi/monetdb/mcl/responses/ResultSetResponse.java
Branch: embedded
Log Message:

After a lot of suffering, finnaly passed all the tests in a MAPI connection! :) 
Now I will port some for the embedded connection, as some features are no 
available on it.


diffs (truncated from 1450 to 300 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -47,6 +47,11 @@ public abstract class MonetConnection ex
 /** The sequence counter */
 private static int SeqCounter = 0;
 
+/**
+ * Gets the current sequence counter.
+ *
+ * @return The current sequence counter
+ */
 public static int GetSeqCounter() {
 return SeqCounter;
 }
@@ -69,6 +74,7 @@ public abstract class MonetConnection ex
 private Map<String,Class> typeMap = new HashMap<String,Class>() {
 private static final long serialVersionUID = 1L; {
 put("inet", MonetINET.class);
+put("url", MonetURL.class);
 }
 };
 
@@ -106,6 +112,15 @@ public abstract class MonetConnection ex
 }
 
 /**
+ * Checks if the conection is embedded or not
+ *
+ * @return If the connection is embedded
+ */
+public boolean isEmbedded() {
+return isEmbedded;
+}
+
+/**
  * Gets the connection's language data.
  *
  * @return The connection's language data
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetINET.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetINET.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetINET.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetINET.java
@@ -29,6 +29,8 @@ import java.sql.SQLOutput;
  * This class allows to retrieve the value of this INET as InetAddress.
  * This is probably meaningful only and only if the netmask is 32.  The
  * getNetmaskBits() method can be used to retrieve the subnet bits.
+ *
+ * @author Fabian Groffen
  */
 public class MonetINET implements SQLData {
 
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
@@ -44,23 +44,11 @@ import java.util.Map;
  * [ "int",9,  0   ]
  * 
  *
- * @author Fabian Groffen, Martin van Dinther
+ * @author Fabian Groffen, Martin van Dinther, Pedro Ferreira
  * @version 0.4
  */
 public class MonetPreparedStatement extends MonetStatement implements 
PreparedStatement {
 
-   /* only parse the date patterns once, use multiple times */
-   /** Format of a timestamp with RFC822 time zone */
-   private static final SimpleDateFormat MTimestampZ = new 
SimpleDateFormat("-MM-dd HH:mm:ss.SSSZ");
-   /** Format of a timestamp */
-   private static final SimpleDateFormat MTimestamp = new 
SimpleDateFormat("-MM-dd HH:mm:ss.SSS");
-   /** Format of a time with RFC822 time zone */
-   private static final SimpleDateFormat MTimeZ = new 
SimpleDateFormat("HH:mm:ss.SSSZ");
-   /** Format of a time */
-   private static final SimpleDateFormat MTime = new 
SimpleDateFormat("HH:mm:ss.SSS");
-   /** Format of a date used by mserver */
-   private static final SimpleDateFormat MDate = new 
SimpleDateFormat("-MM-dd");
-
private final MonetConnection connection;
private final String[] monetdbType;
private final int[] javaType;
@@ -74,6 +62,12 @@ public class MonetPreparedStatement exte
private final int rscolcnt;
private final String[] values;
 
+   private final SimpleDateFormat mTimestampZ;
+   private final SimpleDateFormat mTime

monetdb-java: embedded - Limit the StringBuilder size for the JD...

[Pedro Ferreira]

Changeset: 53ef497672bf for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=53ef497672bf
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
Branch: embedded
Log Message:

Limit the StringBuilder size for the JDBC embedded connection.


diffs (59 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -164,6 +164,13 @@ public abstract class MonetConnection ex
 public abstract int getDefFetchsize();
 
 /**
+ * Gets the initial value for the StringBuilder size.
+ *
+ * @return The initial value for the StringBuilder size
+ */
+public abstract int initialStringBuilderSize();
+
+/**
  * Gets the underlying connection socket timeout.
  *
  * @return The underlying connection socket timeout
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
@@ -191,11 +191,12 @@ public class MonetStatement extends Mone
boolean error = false;
 
BatchUpdateException e = new 
BatchUpdateException("Error(s) occurred while executing the batch, see next 
SQLExceptions for details", "22000", counts);
-   StringBuilder tmpBatch = new 
StringBuilder(connection.getBlockSize());
+   int builderSize = connection.initialStringBuilderSize();
+   StringBuilder tmpBatch = new StringBuilder(builderSize);
String sep = 
connection.getLanguage().getQueryTemplateIndex(2);
for (int i = 0; i < batch.size(); i++) {
String tmp = batch.get(i);
-   if (sep.length() + tmp.length() > 
connection.getBlockSize()) {
+   if (sep.length() + tmp.length() > builderSize) {
// The thing is too big. Way too big. 
Since it won't be optimal anyway, just add it to whatever we
// have and continue.
if (!first) {
@@ -209,7 +210,7 @@ public class MonetStatement extends Mone
first = true;
continue;
}
-   if (tmpBatch.length() + sep.length() + 
tmp.length() >= connection.getBlockSize()) {
+   if (tmpBatch.length() + sep.length() + 
tmp.length() >= builderSize) {
// send and receive
error |= 
internalBatch(tmpBatch.toString(), counts, offset, i + 1, e);
offset = i;
diff --git 
a/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java 
b/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
--- a/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
@@ -191,6 +191,11 @@ public class MapiConnection extends Mone
 return DEF_FETCHSIZE;
 }
 
+@Override
+public int initialStringBuilderSize() {
+return this.getBlockSize();
+}
+
 /**
  * Closes the underlying connection implementation. On a MAPI connection, 
the underlying socket is closed.
  *
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Simplified the the beginning of the JDB...

[Pedro Ferreira]

Changeset: 846cedacec75 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=846cedacec75
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
Branch: embedded
Log Message:

Simplified the the beginning of the JDBC embedded connection, so only the URL 
is necessary to be provided to the Driver Manager to start the connection. The 
JDBC embedded connection URL has the format "jdbc:monetdb:embedded:" 
in contrast to the already existing MAPI one: 
"jdbc:monetdb://[:port]/".


diffs (198 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -106,8 +106,8 @@ public abstract class MonetConnection ex
 this.language = language;
 this.blobIsBinary = blobIsBinary;
 this.clobIsLongChar = clobIsLongChar;
-String embedded = props.getProperty("embedded");
-this.isEmbedded = embedded != null && embedded.equals("true");
+//"instance of" should be cleanner, but this is faster.
+this.isEmbedded = props.getProperty("embedded", 
"false").equals("true");
 }
 
 /**
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
@@ -45,14 +45,15 @@ import java.util.logging.Logger;
  * where [:port] denotes that a port is optional. If not
  * given the default (@JDBC_DEF_PORT@) will be used.
  *
- * @author Fabian Groffen
+ * @author Fabian Groffen, Pedro Ferreira
  * @version @JDBC_MAJOR@.@JDBC_MINOR@ (@JDBC_VER_SUFFIX@)
  */
 final public class MonetDriver implements Driver {
-   // the url kind will be jdbc:monetdb://[:]/
+   // the url kind will be jdbc:monetdb://[:]/ (in a 
MAPI connection)
+   // the url kind will be jdbc:monetdb:embedded: (in an 
Embedded connection)
// Chapter 9.2.1 from Sun JDBC 3.0 specification
/** The prefix of a MonetDB url */
-   private static final String MONETURL = "jdbc:monetdb://";
+   private static final String MONETURL = "jdbc:monetdb:";
/** Major version of this driver */
private static final int DRIVERMAJOR = @JDBC_MAJOR@;
/** Minor version of this driver */
@@ -145,17 +146,7 @@ final public class MonetDriver implement
 
prop = new DriverPropertyInfo("hash", "");
prop.required = false;
-   prop.description = "Force the use of the given hash algorithm 
during challenge response (one of SHA1, MD5, plain)";
-   props.add(prop);
-
-   prop = new DriverPropertyInfo("host", "localhost");
-   prop.required = false;
-   prop.description = "The MonetDB server hostname (MAPI 
connection only)";
-   props.add(prop);
-
-   prop = new DriverPropertyInfo("port", PORT);
-   prop.required = false;
-   prop.description = "The port to connect to the MonetDB server 
(MAPI connection only)";
+   prop.description = "Force the use of the given hash algorithm 
during challenge response (one of SHA1, MD5, plain) (MAPI connection only)";
props.add(prop);
 
prop = new DriverPropertyInfo("so_timeout", "0");
@@ -163,11 +154,6 @@ final public class MonetDriver implement
prop.description = "Defines the maximum time to wait in 
milliseconds on a blocking read socket call (MAPI connection only)"; // this 
corresponds to the Connection.setNetworkTimeout() method introduced in JDBC 4.1
props.add(prop);
 
-   prop = new DriverPropertyInfo("database", "");
-   prop.required = false;
-   prop.description = "The database name to connect (MAPI 
connection only)";
-   props.add(prop);
-
prop = new DriverPropertyInfo("follow_redirects", "true");
prop.required = false;
prop.description = "Whether redirects issued by the server 
should be followed (MAPI connection only)";
@@ -188,16 +174,6 @@ final public class MonetDriver implement
prop.description = "What language to use for MonetDB 
conversations (experts only)";
props.add(prop);
 
-   prop = new DriverPropertyInfo("embedded", "false");
-   prop.required = false;
-   prop.d

monetdb-java: default - Fixed negative number parsing on the Sta...

[Pedro Ferreira]

Changeset: 04c535b05c52 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=04c535b05c52
Modified Files:
src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java
src/main/java/nl/cwi/monetdb/mcl/parser/StartOfHeaderParser.java
Branch: default
Log Message:

Fixed negative number parsing on the StartOfHeaderParser


diffs (54 lines):

diff --git a/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java 
b/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java
--- a/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java
@@ -157,6 +157,10 @@ public class HeaderLineParser extends MC
 * Returns an array of ints containing the values between
 * ',\t' separators.
 *
+* Feb2017 note - This integer parser doesn't have to parse negative
+* numbers, because it is only used to parse column lengths
+* which is always greater than 0.
+*
 * @param chrLine a character array holding the input data
 * @param start where the relevant data starts
 * @param stop where the relevant data stops
@@ -171,7 +175,6 @@ public class HeaderLineParser extends MC
if (chrLine[i] == ',' && chrLine[i + 1] == '\t') {
intValues[elem++] = tmp;
tmp = 0;
-   start = i++;
} else {
tmp *= 10;
// note: don't use Character.isDigit() here, 
because
diff --git a/src/main/java/nl/cwi/monetdb/mcl/parser/StartOfHeaderParser.java 
b/src/main/java/nl/cwi/monetdb/mcl/parser/StartOfHeaderParser.java
--- a/src/main/java/nl/cwi/monetdb/mcl/parser/StartOfHeaderParser.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/parser/StartOfHeaderParser.java
@@ -96,15 +96,18 @@ public class StartOfHeaderParser {
 * @throws MCLParseException if no numeric value could be read
 */
public final int getNextAsInt() throws MCLParseException {
+   boolean positive = true;
pos++;
if (!soh.hasRemaining()) throw
new MCLParseException("unexpected end of string", 
soh.position() - 1);
-   int tmp;
+   int tmp = 0;
char chr = soh.get();
// note: don't use Character.isDigit() here, because
// we only want ISO-LATIN-1 digits
if (chr >= '0' && chr <= '9') {
tmp = (int)chr - (int)'0';
+   } else if(chr == '-') {
+   positive = false;
} else {
throw new MCLParseException("expected a digit", 
soh.position() - 1);
}
@@ -118,7 +121,7 @@ public class StartOfHeaderParser {
}
}
 
-   return tmp;
+   return positive ? tmp : -tmp;
}
 
public final String getNextAsString() throws MCLParseException {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Fixed ResultSet data fetching from the ...

[Pedro Ferreira]

Changeset: a00241382675 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=a00241382675
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
src/main/java/nl/cwi/monetdb/mcl/responses/DataBlockResponse.java
Branch: embedded
Log Message:

Fixed ResultSet data fetching from the JDBC tests output.


diffs (truncated from 547 to 300 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -190,10 +190,9 @@ public abstract class MonetConnection ex
  * @param resultSetType the type of result sets to produce
  * @param resultSetConcurrency the concurrency of result sets to produce
  * @return A ResponseList instance
- * @throws SQLException if an IO exception or a database error occurs
  */
 public abstract ResponseList createResponseList(int fetchSize, int 
maxRows, int resultSetType,
-int resultSetConcurrency) 
throws SQLException;
+int resultSetConcurrency);
 
 /**
  * Releases this Connection object's database and JDBC resources 
immediately instead of waiting for them to be
@@ -1369,7 +1368,7 @@ public abstract class MonetConnection ex
  * @param rstype the type of result sets to produce
  * @param rsconcur the concurrency of result sets to produce
  */
-public ResponseList(int cachesize, int maxrows, int rstype, int 
rsconcur) throws SQLException {
+public ResponseList(int cachesize, int maxrows, int rstype, int 
rsconcur) {
 this.cachesize = cachesize;
 this.maxrows = maxrows;
 this.rstype = rstype;
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
@@ -587,7 +587,50 @@ public class MonetResultSet extends Mone
if(setLastNullValue(columnIndex - 1)) {
return null;
}
-   return (BigDecimal) 
currentBlock.getObjectValue(columnIndex - 1);
+   BigDecimal val;
+   switch (JdbcSQLTypes[columnIndex - 1]) {
+   case Types.DECIMAL:
+   val = (BigDecimal) 
currentBlock.getObjectValue(columnIndex - 1);
+   break;
+   case Types.NUMERIC:
+   BigInteger huge = (BigInteger) 
currentBlock.getValueAsObject(columnIndex - 1);
+   val = 
BigDecimal.valueOf(huge.longValue());
+   break;
+   case Types.BOOLEAN:
+   byte bval = 
currentBlock.getBooleanValue(columnIndex - 1) ? (byte) 1 : (byte) 0;
+   val = new BigDecimal(bval);
+   break;
+   case Types.TINYINT:
+   val = new 
BigDecimal(currentBlock.getByteValue(columnIndex - 1));
+   break;
+   case Types.SMALLINT:
+   val = new 
BigDecimal(currentBlock.getShortValue(columnIndex - 1));
+   break;
+   case Types.INTEGER:
+   val = new 
BigDecimal(currentBlock.getIntValue(columnIndex - 1));
+   break;
+   case Types.BIGINT:
+   val = new 
BigDecimal(currentBlock.getLongValue(columnIndex - 1));
+   break;
+   case Types.REAL:
+   val = new 
BigDecimal(currentBlock.getFloatValue(columnIndex - 1));
+   break;
+   case Types.DOUBLE:
+   val = new 
BigDecimal(currentBlock.getDoubleValue(columnIndex - 1));
+   break;
+   case Types.CHAR:
+   case Types.VARCHAR:
+   case Types.LONGVARCHAR:
+   case Types.CLOB:
+   case Types.BLOB:
+   

monetdb-java: embedded - Made several bug fixes from the tests.

[Pedro Ferreira]

Changeset: d39f656b6614 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=d39f656b6614
Modified Files:
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/MapiConnection.java
src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java

src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiStartOfHeaderParser.java

src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTableHeaderParser.java

src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParser.java

src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParserHelper.java
src/main/java/nl/cwi/monetdb/mcl/responses/DataBlockResponse.java
Branch: embedded
Log Message:

Made several bug fixes from the tests.


diffs (truncated from 643 to 300 lines):

diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -84,6 +84,8 @@ public abstract class MonetConnection ex
 private final boolean clobIsLongChar;
 /** The underlying proticol provided by the connection (MAPI or embedded) 
*/
 protected AbstractProtocol protocol;
+/** Tells if the connection is embedded or not */
+private final boolean isEmbedded;
 
 /**
  * Constructor of a Connection for MonetDB. At this moment the current 
implementation limits itself to storing the
@@ -99,6 +101,8 @@ public abstract class MonetConnection ex
 this.language = language;
 this.blobIsBinary = blobIsBinary;
 this.clobIsLongChar = clobIsLongChar;
+String embedded = props.getProperty("embedded");
+this.isEmbedded = embedded != null && embedded.equals("true");
 }
 
 /**
@@ -183,18 +187,6 @@ public abstract class MonetConnection ex
 public abstract void sendControlCommand(int commandID, int data) throws 
SQLException;
 
 /**
- * Creates a ResponseList.
- *
- * @param fetchSize the nubmer of rows per block in the response list
- * @param maxRows maximum number of rows to allow in the set
- * @param resultSetType the type of result sets to produce
- * @param resultSetConcurrency the concurrency of result sets to produce
- * @return A ResponseList instance
- */
-public abstract ResponseList createResponseList(int fetchSize, int 
maxRows, int resultSetType,
-int resultSetConcurrency);
-
-/**
  * Releases this Connection object's database and JDBC resources 
immediately instead of waiting for them to be
  * automatically released. All Statements created from this Connection 
will be closed when this method is called.
  *
@@ -1498,8 +1490,10 @@ public abstract class MonetConnection ex
  * Change the reply size of the server.  If the given value is 
the same as the current value known
  * to use, then ignore this call.  If it is set to 0 we get a 
prompt after the server sent it's
  * header.
+ *
+ * 2017: For now, in the embedded connection, the value set 
cachesize will be always the default one.
  */
-int size = cachesize == 0 ? 
MonetConnection.this.getDefFetchsize() : cachesize;
+int size = (cachesize != 0 && !isEmbedded) ? cachesize : 
MonetConnection.this.getDefFetchsize();
 size = maxrows != 0 ? Math.min(maxrows, size) : size;
 // don't do work if it's not needed
 if (language == MapiLanguage.LANG_SQL && size != curReplySize 
&&
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
@@ -243,7 +243,7 @@ final public class MonetDriver implement
typeMap.put("double", Types.DOUBLE);
typeMap.put("geometry", Types.OTHER);
typeMap.put("geometrya", Types.OTHER);
-   typeMap.put("hugeint", Types.NUMERIC); //but we will convert to 
java.math.BigInteger
+   typeMap.put("hugeint", Types.NUMERIC);
typeMap.put("inet", Types.OTHER);
typeMap.put("int", Types.INTEGER);
typeMap.put("json", Types.OTHER);
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java 

monetdb-java: embedded - Theere was a bug happening when the use...

[Pedro Ferreira]

Changeset: d4c6a01cc300 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=d4c6a01cc300
Modified Files:
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
src/main/java/nl/cwi/monetdb/mcl/connection/mapi/OldMapiSocket.java
src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java
Branch: embedded
Log Message:

Theere was a bug happening when the user's query was larger than the block 
size. The JDBC client was flushing the stream to the server more than once, so 
it was fixed to flush only when the query ends.


diffs (195 lines):

diff --git 
a/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java 
b/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
--- a/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
+++ b/src/main/java/nl/cwi/monetdb/mcl/connection/mapi/AbstractSocket.java
@@ -52,11 +52,11 @@ public abstract class AbstractSocket imp
 AbstractSocket(String hostname, int port, MapiConnection connection) 
throws IOException {
 this.socket = new Socket(hostname, port);
 this.connection = connection;
-this.bufferIn = ByteBuffer.wrap(new byte[getBlockSize()]);
-this.bufferOut = ByteBuffer.wrap(new byte[getBlockSize()]);
-this.stringsDecoded = CharBuffer.allocate(getBlockSize());
+this.bufferIn = ByteBuffer.wrap(new byte[getFullBlockSize()]);
+this.bufferOut = ByteBuffer.wrap(new byte[getFullBlockSize()]);
+this.stringsDecoded = CharBuffer.allocate(getFullBlockSize());
 this.stringsDecoded.flip();
-this.stringsEncoded = CharBuffer.allocate(getBlockSize());
+this.stringsEncoded = CharBuffer.allocate(getFullBlockSize());
 }
 
 /**
@@ -100,6 +100,13 @@ public abstract class AbstractSocket imp
 }
 
 /**
+ * Gets the underlying socket full block size.
+ *
+ * @return The underlying socket full block size
+ */
+public abstract int getFullBlockSize();
+
+/**
  * Gets the underlying socket block size.
  *
  * @return The underlying socket block size
@@ -134,7 +141,7 @@ public abstract class AbstractSocket imp
  *
  * @throws IOException If an error in the underlying connection happened
  */
-private void readToBuffer() throws IOException {
+private void readToInputBuffer() throws IOException {
 int read = this.readToBufferIn(this.bufferIn);
 if(read == 0) {
 throw new IOException("The server has reached EOF!");
@@ -158,15 +165,18 @@ public abstract class AbstractSocket imp
 boolean found = false;
 char[] sourceArray = this.stringsDecoded.array();
 int sourcePosition = this.stringsDecoded.position();
+int sourceLimit = this.stringsDecoded.limit();
 char[] destinationArray = lineBuffer.array();
 int destinationPosition = 0;
 int destinationLimit = lineBuffer.limit();
 
 while(!found) {
-if(!this.stringsDecoded.hasRemaining()) {
-this.readToBuffer();
+if(sourcePosition >= sourceLimit) {
+this.stringsDecoded.position(sourcePosition);
+this.readToInputBuffer();
 sourceArray = this.stringsDecoded.array();
 sourcePosition = 0;
+sourceLimit = this.stringsDecoded.limit();
 }
 char c = sourceArray[sourcePosition++];
 if(c == '\n') {
@@ -189,9 +199,10 @@ public abstract class AbstractSocket imp
 /**
  * Helper method to write, encode into UTF-8 and flush.
  *
+ * @param toFlush A boolean indicating to flush the underlying stream or 
not
  * @throws IOException If an error in the underlying connection happened
  */
-private void flushOutputCharBuffer() throws IOException {
+private void writeToOutputBuffer(boolean toFlush) throws IOException {
 this.stringsEncoded.flip();
 this.utf8Encoder.reset();
 this.utf8Encoder.encode(this.stringsEncoded, this.bufferOut, true);
@@ -201,23 +212,28 @@ public abstract class AbstractSocket imp
 if(written == 0) {
 throw new IOException("The query could not be sent to the 
server!");
 } else {
-this.flush();
+if(toFlush) {
+this.flush();
+}
 }
 }
 
 /**
  * Writes a String line into the underlying socket.
  *
+ * @param line The line to write in the socket
  * @throws IOException If an error in the underlying connection happened
  */
 private void writeNextBlock(String line) throws IOException {
 int limit = line.length();
 int destinationPosition = this.stringsEncoded.position();
+int destinationCapacity = this.stringsEncoded.capacity();
 char[] destinationArray = this.stringsEncoded.array();
 
 for (int i = 0; i < limit; i++) {
-

MonetDB: iot - Added bug test for tuple windows correctness

[Pedro Ferreira]

Changeset: b29c9d42646c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b29c9d42646c
Added Files:
sql/backends/monet5/iot/Tests/bug06.sql
Branch: iot
Log Message:

Added bug test for tuple windows correctness


diffs (26 lines):

diff --git a/sql/backends/monet5/iot/Tests/bug06.sql 
b/sql/backends/monet5/iot/Tests/bug06.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/iot/Tests/bug06.sql
@@ -0,0 +1,21 @@
+CREATE SCHEMA iot;
+SET SCHEMA iot;
+SET OPTIMIZER = 'iot_pipe';
+
+CREATE STREAM TABLE temperature (t TIMESTAMP, sensor INT, val DECIMAL);
+CALL window('iot', 'temperature', 3);
+CREATE TABLE results (minimum TIMESTAMP, tuples INT, average DECIMAL);
+
+CREATE PROCEDURE testing()
+BEGIN
+   INSERT INTO results SELECT MIN(t), COUNT(*), AVG(val) FROM temperature;
+END;
+
+CALL query('iot', 'testing');
+
+INSERT INTO temperature VALUES (now(), 1, 1), (now(), 2, 2), (now(), 3, 3), 
(now(), 4, 4);
+
+SELECT * FROM results; /* should have only 1 row, where "tuples" column value 
is 3 */
+
+CALL getwindow('iot','temperature'); /* should not hang! */
+
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Major change in the Datablock response....

[Pedro Ferreira]

 // read the next line (can be prompt, new result, 
error, etc.) before we start the loop over
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
@@ -48,7 +48,7 @@ import java.util.logging.Logger;
  * @author Fabian Groffen, Pedro Ferreira
  * @version @JDBC_MAJOR@.@JDBC_MINOR@ (@JDBC_VER_SUFFIX@)
  */
-final public class MonetDriver implements Driver {
+public final class MonetDriver implements Driver {
// the url kind will be jdbc:monetdb://[:]/ (in a 
MAPI connection)
// the url kind will be jdbc:monetdb:embedded: (in an 
Embedded connection)
// Chapter 9.2.1 from Sun JDBC 3.0 specification
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
@@ -8,37 +8,23 @@
 
 package nl.cwi.monetdb.jdbc;
 
-import nl.cwi.monetdb.mcl.responses.DataBlockResponse;
+import nl.cwi.monetdb.mcl.protocol.ProtocolException;
+import nl.cwi.monetdb.mcl.responses.AbstractDataBlockResponse;
 import nl.cwi.monetdb.mcl.responses.ResultSetResponse;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.math.BigDecimal;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.sql.Array;
-import java.sql.Blob;
-import java.sql.Clob;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.Date;
-import java.sql.NClob;
-import java.sql.Ref;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.RowId;
-import java.sql.SQLData;
-import java.sql.SQLException;
-import java.sql.SQLFeatureNotSupportedException;
-import java.sql.SQLInput;
-import java.sql.SQLWarning;
-import java.sql.SQLXML;
-import java.sql.Statement;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.*;
+import java.sql.*;
+import java.util.Calendar;
+import java.util.Map;
+import java.util.UUID;
 
 /**
  * A ResultSet suitable for the MonetDB database.
@@ -84,12 +70,10 @@ public class MonetResultSet extends Mone
private int concurrency = CONCUR_READ_ONLY;
/** The warnings for this ResultSet object */
private SQLWarning warnings;
-   /** whether the last read field (via some getXyz() method) was NULL */
-   private boolean lastReadWasNull = true;
/** Just a dummy variable to keep store the fetchsize set. */
private int fetchSize;
/** The current row's values */
-   private DataBlockResponse currentBlock;
+   private AbstractDataBlockResponse currentBlock;
 
/**
 * Main constructor backed by the given Header.
@@ -152,11 +136,6 @@ public class MonetResultSet extends Mone
this.JdbcSQLTypes = JdbcSQLTypes;
}
 
-   private boolean setLastNullValue(int columnIndex) {
-   this.lastReadWasNull = 
currentBlock.checkValueIsNull(columnIndex);
-   return this.lastReadWasNull;
-   }
-
//== methods of interface ResultSet
 
// Chapter 14.2.2 Sun JDBC 3.0 Specification
@@ -303,23 +282,29 @@ public class MonetResultSet extends Mone
@Override
public InputStream getAsciiStream(int columnIndex) throws SQLException {
 try {
-if(setLastNullValue(columnIndex - 1)) {
-return null;
-}
+   InputStream res = null;
 switch (JdbcSQLTypes[columnIndex - 1]) {
+case Types.CLOB:
+   Clob cl = getClob(columnIndex);
+if(cl != null) {
+res = cl.getAsciiStream();
+}
+   break;
 case Types.BLOB:
-return getClob(columnIndex).getAsciiStream();
-case Types.CLOB:
-return getClob(columnIndex).getAsciiStream();
 case Types.LONGVARBINARY:
 case Types.CHAR:
 case Types.VARCHAR:
 case Types.LONGVARCHAR:
-return new ByteArrayInputStream(getBytes(columnIndex));
+byte[] bytes = getBytes(columnIndex);
+if(bytes != null) {
+res = new ByteArrayInputStream(getBytes(columnIndex));
+}
+   break;
 default:
 throw new SQLException("Conversion from " + 
types[columnIndex - 1] +
 "

MonetDB: iot - Merged

[Pedro Ferreira]

Changeset: 94b98320b6bc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=94b98320b6bc
Added Files:
buildtools/autogen/autogen/filesplit.py
monetdb5/mal/Tests/tst023.malC
monetdb5/mal/Tests/tst023.stable.err
monetdb5/mal/Tests/tst023.stable.out
sql/backends/monet5/iot/Tests/iot01.sql
sql/backends/monet5/iot/Tests/iot01.stable.err
sql/backends/monet5/iot/Tests/iot01.stable.out
sql/backends/monet5/iot/Tests/iot11.sql
sql/backends/monet5/iot/Tests/iot11.stable.err
sql/backends/monet5/iot/Tests/iot11.stable.out
sql/benchmarks/tpch/Tests/02-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/02-plan.stable.out.32bit
sql/benchmarks/tpch/Tests/13-plan.stable.out.32bit
sql/benchmarks/tpch/Tests/16-plan.stable.out.32bit

sql/test/BugTracker-2010/Tests/offset_limited_32bit.SF-2950579.stable.err.32bit

sql/test/BugTracker-2010/Tests/offset_limited_32bit.SF-2950579.stable.out.32bit
sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.sql

sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.stable.err

sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.stable.out
sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.sql
sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.stable.err
sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.stable.out
sql/test/BugTracker-2016/Tests/simpletest.sql
sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.sql

sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.stable.err

sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.stable.out
sql/test/orderidx/Tests/simpletable.stable.out.32bit
sql/test/orderidx/Tests/smalltable.stable.out.32bit
sql/test/pg_regress/Tests/oid.stable.out.32bit
sql/test/pg_regress/Tests/without_oid.stable.out.32bit
Removed Files:
sql/benchmarks/tpch/Tests/01-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/04-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/13-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/22-explain.stable.out.32bit

sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out.32bit

sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552.stable.out.32bit

sql/test/BugTracker-2010/Tests/prepare_stmt.Bug-2533.Bug-2534.stable.out.32bit

sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out.32bit
sql/test/BugTracker-2016/Tests/rename_exps.Bug-3974.stable.out.32bit
sql/test/Dependencies/Tests/Dependencies.stable.out.int128
Modified Files:
buildtools/autogen/autogen/am.py
buildtools/autogen/autogen/codegen.py
buildtools/autogen/autogen/msc.py
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/exports.stable.out
common/stream/stream.c
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_calc.c
gdk/gdk_calc.h
gdk/gdk_cross.c
gdk/gdk_storage.c
gdk/gdk_utils.c
gdk/gdk_value.c
java/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java
java/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java
java/src/main/java/nl/cwi/monetdb/mcl/parser/TupleLineParser.java

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows.single

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single
monetdb5/mal/Tests/All
monetdb5/mal/Tests/tst031.malC
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_debugger.h
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_module.c
monetdb5/mal/mal_profiler.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/json.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/mal/Tests/inspect05.stable.out
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/mkey.c
monetdb5/modules/mal/tablet.c
monetdb5/optimizer/Tests/manifold2.stable.out.single
monetdb5/optimizer/Tests/tst4000.malC
monetdb5/optimizer/Tests/tst4003.malC
monetdb5/optimizer/Tests/tst4010.malC

MonetDB: iot - Updated ais tests and imports

[Pedro Ferreira]

Changeset: 287cab123693 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=287cab123693
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/customthreading.py
clients/iotclient/src/main.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais03.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais04.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais05.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais08.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais09.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
sql/backends/monet5/iot/Tests/bug06.sql
Branch: iot
Log Message:

Updated ais tests and imports


diffs (truncated from 605 to 300 lines):

diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -4,8 +4,8 @@ jsonschema>=2.5.1
 python-dateutil>=2.5.3
 python-monetdb>=11.19.3.2
 pytz>=2016.4
-requests>=2.10.0
-Sphinx>=1.4.4
+requests>=2.11.1
+Sphinx>=1.4.6
 sphinx-rtd-theme>=0.1.9
 tornado>=3.2.2
 tzlocal>=1.2.2
diff --git a/clients/iotapi/src/Streams/datatypes.py 
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,9 +1,9 @@
 import struct
-
 from abc import ABCMeta, abstractmethod
 from datetime import date, time, datetime
+from os import SEEK_END
+
 from dateutil.relativedelta import relativedelta
-from os import SEEK_END
 
 LITTLE_ENDIAN_ALIGNMENT = '<'  # for now is little-endian for Intel CPU's
 
diff --git a/clients/iotapi/src/Streams/streampolling.py 
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -1,12 +1,14 @@
 from collections import OrderedDict, defaultdict
 from json import dumps
+
+from Settings.iotlogger import add_log
+from Settings.mapiconnection import mapi_get_database_streams
+from Utilities.customthreading import PeriodicalThread
+
 from .datatypes import TextType, LimitedTextType, SmallIntegerType, 
HugeIntegerType, FloatType, DecimalType,\
 BooleanType, DateType, TimeType, TimestampType, INetType, UUIDType, 
IntervalType
 from .streams import IOTStream
 from .streamscontext import Streams_Context
-from Settings.iotlogger import add_log
-from Settings.mapiconnection import mapi_get_database_streams
-from Utilities.customthreading import PeriodicalThread
 
 Switcher = [{'types': ['clob', 'url'], 'class': TextType},
 {'types': ['char', 'varchar'], 'class': LimitedTextType},
diff --git a/clients/iotapi/src/Streams/streams.py 
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -1,11 +1,11 @@
 import os
+from collections import OrderedDict
 
-from collections import OrderedDict
-from watchdog.events import FileSystemEventHandler, DirCreatedEvent, 
DirDeletedEvent
-from watchdog.observers import Observer
 from Settings.filesystem import get_baskets_base_location
 from Utilities.readwritelock import RWLock
 from WebSockets.websockets import notify_stream_inserts_to_clients
+from watchdog.events import FileSystemEventHandler, DirCreatedEvent, 
DirDeletedEvent
+from watchdog.observers import Observer
 
 
 def represents_int(s):
diff --git a/clients/iotapi/src/Streams/streamscontext.py 
b/clients/iotapi/src/Streams/streamscontext.py
--- a/clients/iotapi/src/Streams/streamscontext.py
+++ b/clients/iotapi/src/Streams/streamscontext.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+
 from Utilities.readwritelock import RWLock
 from WebSockets.websockets import unsubscribe_removed_streams
 
diff --git a/clients/iotapi/src/Utilities/customthreading.py 
b/clients/iotapi/src/Utilities/customthreading.py
--- 

MonetDB: iot - Removed Flask log messages

[Pedro Ferreira]

Changeset: e5fc524d0512 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e5fc524d0512
Modified Files:
clients/iotclient/src/Flask/app.py
Branch: iot
Log Message:

Removed Flask log messages


diffs (16 lines):

diff --git a/clients/iotclient/src/Flask/app.py 
b/clients/iotclient/src/Flask/app.py
--- a/clients/iotclient/src/Flask/app.py
+++ b/clients/iotclient/src/Flask/app.py
@@ -1,8 +1,12 @@
+import logging
+
 from flask import Flask
 from flask_restful import Api
 
 from .restresources import StreamInput, StreamsInfo, StreamsHandling
 
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)  # disable logging messages to the stderr
 
 def start_flask_iot_app(host, port):
 iot_app = Flask(__name__)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Merged with default

[Pedro Ferreira]

Changeset: ec20c6a000a1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec20c6a000a1
Added Files:
monetdb5/optimizer/opt_emptybind.c
monetdb5/optimizer/opt_emptybind.h
monetdb5/optimizer/opt_jit.c
monetdb5/optimizer/opt_jit.h
sql/test/BugTracker-2016/Tests/like_in_non_sys_schema.Bug-4062.sql

sql/test/BugTracker-2016/Tests/like_in_non_sys_schema.Bug-4062.stable.err

sql/test/BugTracker-2016/Tests/like_in_non_sys_schema.Bug-4062.stable.out
sql/test/BugTracker-2016/Tests/simpletest.stable.err
sql/test/BugTracker-2016/Tests/simpletest.stable.out
Removed Files:
sql/test/pg_regress/Tests/oid.stable.err.Windows
Modified Files:
NT/installer64/MonetDB5-SQL-Installer.vdproj
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.SQL.py
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/exports.stable.out
configure.ag
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_atoms.c
gdk/gdk_bbp.c
gdk/gdk_calc.c
gdk/gdk_calc_compare.h
gdk/gdk_firstn.c
gdk/gdk_group.c
gdk/gdk_hash.h
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_private.h
gdk/gdk_project.c
gdk/gdk_select.c
gdk/gdk_storage.c
gdk/gdk_system.c
gdk/gdk_system.h
gdk/gdk_system_private.h
gdk/gdk_utils.c
gdk/gdk_value.c
geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
geom/sql/40_geom.sql
java/ChangeLog
java/ChangeLog.Jun2016
java/src/main/java/nl/cwi/monetdb/client/JdbcClient.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java
java/src/main/java/nl/cwi/monetdb/util/CmdLineOpts.java
java/src/main/java/nl/cwi/monetdb/util/OptionsException.java
java/src/main/java/nl/cwi/monetdb/util/SQLExporter.java
java/tests/BugDatabaseMetaData_Bug_3356.java
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single
monetdb5/extras/mal_optimizer_template/opt_sql_append.c
monetdb5/extras/rapi/Tests/rapi02.malC
monetdb5/extras/rapi/Tests/rapi02.stable.out
monetdb5/mal/Tests/tst901a.stable.out
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_builder.h
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_function.h
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_interpreter.h
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_listing.h
monetdb5/mal/mal_module.c
monetdb5/mal/mal_namespace.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_private.h
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_stack.h
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/kernel/aggr.c
monetdb5/modules/kernel/aggr.mal
monetdb5/modules/kernel/aggr.mal.sh
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/algebra.h
monetdb5/modules/kernel/algebra.mal
monetdb5/modules/kernel/bat5.c
monetdb5/modules/kernel/bat5.h
monetdb5/modules/kernel/bat5.mal
monetdb5/modules/kernel/microbenchmark.c
monetdb5/modules/kernel/microbenchmark.h
monetdb5/modules/kernel/status.c
monetdb5/modules/mal/00_calc_hge.mal
monetdb5/modules/mal/00_mkey_hge.mal
monetdb5/modules/mal/01_batcalc.mal
monetdb5/modules/mal/Tests/inspect05.stable.out
monetdb5/modules/mal/Tests/inspect05.stable.out.int128
monetdb5/modules/mal/Tests/remote03.stable.err
monetdb5/modules/mal/batExtensions.c
monetdb5/modules/mal/batcalc.c
monetdb5/modules/mal/groupby.c
monetdb5/modules/mal/inspect.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/mkey.c
monetdb5/modules/mal/mkey.h
monetdb5/modules/mal/mkey.mal
monetdb5/modules/mal/projectionpath.c

monetdb-java: embedded - Arranged code in packages and starting ...

[Pedro Ferreira]

Changeset: 3a19ebf83af6 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=3a19ebf83af6
Added Files:
src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedInstance.java
src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractColumn.java
src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractResultTable.java
src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java

src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSet.java

src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSetColumn.java
src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSetRows.java
src/main/java/nl/cwi/monetdb/embedded/resultset/UpdateResultSet.java

src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableRemover.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableUpdater.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTableColumn.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowRemover.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowUpdater.java
Removed Files:
src/main/java/nl/cwi/monetdb/embedded/AbstractColumn.java
src/main/java/nl/cwi/monetdb/embedded/AbstractStatementResult.java
src/main/java/nl/cwi/monetdb/embedded/EmbeddedPreparedStatement.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedBlob.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedConnection.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedDatabase.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedException.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedInstance.java
src/main/java/nl/cwi/monetdb/embedded/MonetDBToJavaMapping.java
src/main/java/nl/cwi/monetdb/embedded/QueryResultSet.java
src/main/java/nl/cwi/monetdb/embedded/QueryResultSetColumn.java
src/main/java/nl/cwi/monetdb/embedded/QueryResultSetRows.java
src/main/java/nl/cwi/monetdb/embedded/UpdateResultSet.java
Branch: embedded
Log Message:

Arranged code in packages and starting tables integration.


diffs (truncated from 1272 to 300 lines):

diff --git a/src/main/java/nl/cwi/monetdb/embedded/AbstractStatementResult.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
rename from src/main/java/nl/cwi/monetdb/embedded/AbstractStatementResult.java
rename to 
src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
--- a/src/main/java/nl/cwi/monetdb/embedded/AbstractStatementResult.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
@@ -6,23 +6,23 @@
  * Copyright 2016 MonetDB B.V.
  */
 
-package nl.cwi.monetdb.embedded;
+package nl.cwi.monetdb.embedded.env;
 
 import java.io.Closeable;
 
 /**
- * The base class for a query result.
+ * The base class for a pending statement to a connection.
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public abstract class AbstractStatementResult implements Closeable {
+public abstract class AbstractConnectionResult implements Closeable {
 
 /**
  * The corresponding connection of this result.
  */
 private final MonetDBEmbeddedConnection connection;
 
-protected AbstractStatementResult(MonetDBEmbeddedConnection connection) {
+protected AbstractConnectionResult(MonetDBEmbeddedConnection connection) {
 this.connection = connection;
 }
 
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedConnection.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
rename from src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedConnection.java
rename to 
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
--- a/src/main/java/nl/cwi/monetdb/embedded/MonetDBEmbeddedConnection.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
@@ -6,7 +6,12 @@
  * Copyright 2016 MonetDB B.V.
  */
 
-package nl.cwi.monetdb.embedded;
+package nl.cwi.monetdb.embedded.env;
+
+import nl.cwi.monetdb.embedded.resultset.EmbeddedPreparedStatement;
+imp

monetdb-java: embedded - Change some Java Class - BAT mappings

[Pedro Ferreira]

es = 
this.getMonetDBTableInternal(this.connectionPointer, schemaName, tableName);
 results.add(res);
 return res;
 }
@@ -216,7 +189,7 @@ public class MonetDBEmbeddedConnection {
  *
  * @param schemaName The schema of the table
  * @param tableName The name of the table
- * @return A MonetDBTable instance with currentColumns details
+ * @return A MonetDBTable instance with column details
  * @throws MonetDBEmbeddedException If an error in the database occurred
  */
 /*public MonetDBTable getMonetDBTableAsync(String schemaName, String 
tableName) throws MonetDBEmbeddedException {
@@ -301,9 +274,7 @@ public class MonetDBEmbeddedConnection {
 /**
  * Removes a query result from this connection.
  */
-protected void removeQueryResult(AbstractConnectionResult res) {
-this.results.remove(res);
-}
+protected void removeQueryResult(AbstractConnectionResult res) { 
this.results.remove(res); }
 
 /**
  * Internal implementation of sendUpdate.
@@ -317,6 +288,12 @@ public class MonetDBEmbeddedConnection {
 private native QueryResultSet sendQueryInternal(long connectionPointer, 
String query, boolean execute)
 throws MonetDBEmbeddedException;
 
+/**
+ * Internal implementation of getMonetDBTable.
+ */
+private native MonetDBTable getMonetDBTableInternal(long 
connectionPointer, String schemaName, String tableName)
+throws MonetDBEmbeddedException;
+
 /*private native EmbeddedPreparedStatement 
createPreparedStatementInternal(long connectionPointer, String query)
 throws MonetDBEmbeddedException;*/
 
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java 
b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
--- a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
@@ -1,5 +1,6 @@
 package nl.cwi.monetdb.embedded.mapping;
 
+import java.io.Serializable;
 import java.util.Arrays;
 
 /**
@@ -7,7 +8,7 @@ import java.util.Arrays;
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public class MonetDBEmbeddedBlob {
+public class MonetDBEmbeddedBlob implements Serializable {
 
 /**
  * The BLOB's content as a Java byte array.
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java 
b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
--- a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
@@ -10,8 +10,8 @@ package nl.cwi.monetdb.embedded.mapping;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.net.InetAddress;
-import java.net.URI;
+import java.net.Inet4Address;
+import java.net.URL;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
@@ -30,7 +30,7 @@ public enum MonetDBToJavaMapping {
 Hugeint(BigInteger.class), Decimal(BigDecimal.class), Real(Float.class), 
Double(Double.class),
 MonthInterval(Integer.class), SecondInterval(Long.class), 
Time(Time.class), TimeTz(Time.class), Date(Date.class),
 Timestamp(Timestamp.class), TimestampTz(Timestamp.class), 
Blob(MonetDBEmbeddedBlob.class),
-Geometry(MonetDBEmbeddedBlob.class), GeometryA(MonetDBEmbeddedBlob.class), 
URL(URI.class), Inet(InetAddress.class),
+Geometry(MonetDBEmbeddedBlob.class), GeometryA(MonetDBEmbeddedBlob.class), 
URL(URL.class), Inet(Inet4Address.class),
 JSON(MonetDBEmbeddedBlob.class), UUID(UUID.class);
 
 /**
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
 
b/src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
--- 
a/src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
+++ 
b/src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
@@ -17,8 +17,8 @@ import nl.cwi.monetdb.embedded.utils.Str
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
-import java.net.InetAddress;
-import java.net.URI;
+import java.net.Inet4Address;
+import java.net.URL;
 import java.sql.PreparedStatement;
 import java.sql.Time;
 import java.sql.Timestamp;
@@ -72,8 +72,8 @@ public class EmbeddedPreparedStatement {
 DirectMappingClasses.add(Float.class);
 DirectMappingClasses.add(Double.class);
 DirectMappingClasses.add(MonetDBEmbeddedBlob.class);
-DirectMappingClasses.add(URI.class);
-DirectMappingClasses.add(InetAddress.class);
+DirectMappingClasses.add(URL.class);
+DirectMappingClasses.add(Inet4Address.class);
 DirectMappingClasses.add(UUID.class);
 }
 
@@ -366,6 +366,7 @@ public class EmbeddedPreparedStatement {
 /**
  * Creates the SQL String from the parsed parameters (adapted from the 
JDBC dr

monetdb-java: embedded - Finished Java code for CRUD operations ...

[Pedro Ferreira]

Changeset: dfea8468cd1a for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=dfea8468cd1a
Added Files:
src/main/java/nl/cwi/monetdb/embedded/utils/StringEscaper.java
Modified Files:
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractColumn.java
src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractResultTable.java
src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java

src/main/java/nl/cwi/monetdb/embedded/resultset/EmbeddedPreparedStatement.java
src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSet.java

src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSetColumn.java
src/main/java/nl/cwi/monetdb/embedded/resultset/QueryResultSetRows.java
src/main/java/nl/cwi/monetdb/embedded/resultset/UpdateResultSet.java

src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableRemover.java
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableUpdater.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTableColumn.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowRemover.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowUpdater.java
Branch: embedded
Log Message:

Finished Java code for CRUD operations on tables and the documentation.


diffs (truncated from 1335 to 300 lines):

diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
@@ -8,10 +8,10 @@
 
 package nl.cwi.monetdb.embedded.env;
 
-import nl.cwi.monetdb.embedded.resultset.EmbeddedPreparedStatement;
-import nl.cwi.monetdb.embedded.resultset.QueryResultSet;
-import nl.cwi.monetdb.embedded.resultset.QueryResultSetColumn;
-import nl.cwi.monetdb.embedded.resultset.UpdateResultSet;
+import nl.cwi.monetdb.embedded.resultset.*;
+import nl.cwi.monetdb.embedded.tables.MonetDBTable;
+import nl.cwi.monetdb.embedded.tables.MonetDBTableColumn;
+import nl.cwi.monetdb.embedded.utils.StringEscaper;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -44,7 +44,7 @@ public class MonetDBEmbeddedConnection {
  * @throws MonetDBEmbeddedException If an error in the database occurred
  */
 public String getCurrentSchema() throws MonetDBEmbeddedException {
-QueryResultSet eqr = this.sendQuery("select current_schema from 
sys.var();");
+QueryResultSet eqr = this.sendQuery("SELECT current_schema FROM 
sys.var();");
 QueryResultSetColumn col = eqr.getColumn(0);
 String res = col.fetchFirstNColumnValues(1)[0];
 eqr.close();
@@ -54,12 +54,12 @@ public class MonetDBEmbeddedConnection {
 /**
  * Sets the current schema on the connection.
  *
- * @param currentSchema Java String with the name of the schema
+ * @param newSchema Java String with the name of the schema
  * @throws MonetDBEmbeddedException If an error in the database occurred
  */
-public void setCurrentSchema(String currentSchema) throws 
MonetDBEmbeddedException {
-String valueToSubmit = "'" + currentSchema.replaceAll("", 
"").replaceAll("'", "'") + "';";
-this.sendUpdate("SET SCHEMA " + valueToSubmit).close();
+public void setCurrentSchema(String newSchema) throws 
MonetDBEmbeddedException {
+newSchema = StringEscaper.SQLStringEscape(newSchema);
+this.sendUpdate("SET SCHEMA " + newSchema + ";").close();
 }
 
 /**
@@ -152,12 +152,12 @@ public class MonetDBEmbeddedConnection {
  * @return An instance of EmbeddedPreparedStatement
  * @throws MonetDBEmbeddedException If an error in the database occurred
  */
-public EmbeddedPreparedStatement createPreparedStatement(String query) 
throws MonetDBEmbeddedException {
+/*public EmbeddedPreparedStatement createPreparedStatement(String query) 
throws MonetDBEmbeddedException {
 if (!query.endsWith(";")) {
 query += ";";
 }
 return this.createPreparedStatementInternal(this.connectionPointer, 
query);
-}
+}*/
 
 /**
  * Creates a prepared query statement likewise the PreparedStatement in 
JDBC asynchronously.
@@ 

monetdb-java: embedded - Starting to test Java to BAT conversion...

[Pedro Ferreira]

Changeset: 8353929359d6 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=8353929359d6
Added Files:
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableCursor.java
Removed Files:
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java
Modified Files:
src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTableColumn.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowRemover.java
src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
Branch: embedded
Log Message:

Starting to test Java to BAT conversion. Let the debugging games begin!


diffs (truncated from 407 to 300 lines):

diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java 
b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
--- a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBToJavaMapping.java
@@ -25,6 +25,7 @@ import java.util.UUID;
  */
 public enum MonetDBToJavaMapping {
 
+/* PLEASE Don't change these enum values order!! */
 Boolean(Boolean.class), Char(String.class), Varchar(String.class), 
Clob(String.class), Oid(Long.class),
 Tinyint(Byte.class), Smallint(Short.class), Int(Integer.class), 
Wrd(Long.class), Bigint(Long.class),
 Hugeint(BigInteger.class), Decimal(BigDecimal.class), Real(Float.class), 
Double(Double.class),
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableCursor.java
rename from 
src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java
rename to src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableCursor.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableIterator.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableCursor.java
@@ -5,7 +5,7 @@ package nl.cwi.monetdb.embedded.tables;
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public interface IMonetDBTableIterator extends IMonetDBTableBaseIterator {
+public interface IMonetDBTableCursor extends IMonetDBTableBaseIterator {
 
 /**
  * The business logic for the iterator.
diff --git a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
@@ -4,6 +4,7 @@ import nl.cwi.monetdb.embedded.env.Monet
 import nl.cwi.monetdb.embedded.mapping.AbstractColumn;
 import nl.cwi.monetdb.embedded.mapping.AbstractResultTable;
 import nl.cwi.monetdb.embedded.env.MonetDBEmbeddedConnection;
+import nl.cwi.monetdb.embedded.mapping.MonetDBRow;
 import nl.cwi.monetdb.embedded.resultset.QueryResultSet;
 import nl.cwi.monetdb.embedded.resultset.QueryResultSetColumn;
 
@@ -35,6 +36,11 @@ public class MonetDBTable extends Abstra
  */
 private long connectionPointer;
 
+private final int[] columnsJavaIndexes;
+private final int[] columnsMonetDBIndexes;
+private final String[] columnsNames;
+private final Class[] columnsClasses;
+
 public MonetDBTable(MonetDBEmbeddedConnection connection, long 
connectionPointer, String schemaName,
 String tableName, MonetDBTableColumn[] columns) {
 super(connection);
@@ -42,6 +48,18 @@ public class MonetDBTable extends Abstra
 this.schemaName = schemaName;
 this.tableName = tableName;
 this.columns = columns;
+this.columnsJavaIndexes = new int[columns.length];
+this.columnsMonetDBIndexes = new int[columns.length];
+this.columnsNames = new String[columns.length];
+this.columnsClasses = new Class[columns.length];
+int i = 0;
+for(MonetDBTableColumn col : this.columns) {
+this.columnsJavaIndexes[i] = col.getInternalMonetDBTypeIndex();
+this.columnsMonetDBIndexes[i] = col.getMapping().ordinal();
+this.columnsNames[i] = col.getColumnName();
+this.columnsClasses[i] = col.getMapping().getJavaClass();
+i++;
+}
 }
 
 @Override
@@ -114,32 +132,69 @@ public class MonetDBTable extends Abstra
 }
 
 /**
- * Iterate over the table using a {@link 
nl.cwi.monetdb.embedded.tables.IMonetDBTableIterator} instance.
+ * Private method to check the limits of iteration.
  *
- * @param iterator The iterator with the business logic
+ * @param iterator The iterator to check
+ * @return An integer array with the limits fixed
+ */
+private int[] checkIterator(IMonetDBTableBaseIterator itera

monetdb-java: embedded - Fixed row and column indexing

[Pedro Ferreira]

Changeset: 8217f77fcf6b for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=8217f77fcf6b
Modified Files:
src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractRowSet.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
Branch: embedded
Log Message:

Fixed row and column indexing


diffs (112 lines):

diff --git a/src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractRowSet.java 
b/src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractRowSet.java
--- a/src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractRowSet.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/mapping/AbstractRowSet.java
@@ -20,8 +20,8 @@ public abstract class AbstractRowSet {
 
 protected AbstractRowSet(MonetDBToJavaMapping[] mappings, Object[][] rows) 
{
 this.mappings = mappings;
-this.rows = new MonetDBRow[mappings.length];
-for(int i = 0 ; i < mappings.length ; i++) {
+this.rows = new MonetDBRow[rows.length];
+for(int i = 0 ; i < rows.length ; i++) {
 this.rows[i] = new MonetDBRow(this, rows[i]);
 }
 }
diff --git a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
@@ -82,8 +82,9 @@ public class MonetDBTable extends Abstra
 try {
 String query = "SELECT COUNT(*) FROM " + this.schemaName + "." + 
this.tableName + ";";
 QueryResultSet eqr = this.getConnection().sendQuery(query);
-QueryResultSetColumn eqc = eqr.getColumn(0);
-res = eqc.fetchFirstNColumnValues(1)[0];
+QueryResultSetColumn eqc = eqr.getColumn(0);
+res = eqc.fetchFirstNColumnValues(1)[0].intValue();
+eqr.close();
 } catch (MonetDBEmbeddedException ex) {
 }
 return res;
@@ -139,19 +140,16 @@ public class MonetDBTable extends Abstra
  */
 private int[] checkIterator(IMonetDBTableBaseIterator iterator) {
 int[] res = {iterator.getFirstRowToIterate(), 
iterator.getLastRowToIterate()};
-if(res[0] == res[1]) {
-throw new ArrayIndexOutOfBoundsException("Iterating over 0 rows?");
-}
 if(res[1] < res[0]) {
 int aux = res[0];
 res[0] = res[1];
 res[0] = aux;
 }
-if (res[0] < 0) {
-res[0] = 0;
+if (res[0] < 1) {
+res[0] = 1;
 }
 int numberOfRows = this.getNumberOfRows();
-if (res[1] > numberOfRows) {
+if (res[1] >= numberOfRows) {
 res[1] = numberOfRows;
 }
 return res;
@@ -166,9 +164,9 @@ public class MonetDBTable extends Abstra
  */
 public int iterateTable(IMonetDBTableCursor cursor) throws 
MonetDBEmbeddedException {
 int[] limits = this.checkIterator(cursor);
-int res = 0, total = limits[1] - limits[0];
+int res = 0, total = limits[1] - limits[0] + 1;
 String query = new StringBuffer("SELECT * FROM 
").append(this.schemaName).append(".").append(this.tableName)
-.append(" LIMIT ").append(total).append(" OFFSET 
").append(limits[0]).append(";").toString();
+.append(" LIMIT ").append(total).append(" OFFSET 
").append(limits[0] - 1).append(";").toString();
 
 QueryResultSet eqr = this.getConnection().sendQuery(query);
 MonetDBRow[] array = eqr.fetchAllRowValues().getAllRows();
@@ -326,13 +324,13 @@ public class MonetDBTable extends Abstra
  */
 public int appendColumns(Object[][] columns) throws 
MonetDBEmbeddedException {
 int numberOfRows = columns[0].length, numberOfColumns = 
this.getNumberOfColumns();
+if (columns.length != numberOfColumns) {
+throw new ArrayStoreException("The number of columns differs from 
the table's number of columns!");
+}
 if(numberOfRows == 0) {
 throw new ArrayStoreException("Appending 0 rows?");
 }
-if (columns.length != numberOfColumns) {
-throw new ArrayStoreException("The number of columns differs from 
the table's number of columns!");
-}
-for (int i = 0; i < numberOfRows; i++) {
+for (int i = 0; i < numberOfColumns; i++) {
 if(columns[i].length != numberOfRows) {
 throw new ArrayStoreException("The number of rows in each 
column is not consistent!");
 }
diff --git a/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
@@ -36,7 +36,7 @@ public class RowIterator extends Abstrac
 

monetdb-java: embedded - More cleanup, finished null values mapp...

[Pedro Ferreira]

@ public class MonetDBEmbeddedDatabase {
 /**
  * Get the database farm directory.
  *
+ * @throws MonetDBEmbeddedException If the database is not running
  * @return A String representing the farm directory
  */
-public static String GetDatabaseDirectory() { return 
MonetDBEmbeddedDatabase.databaseDirectory; }
+public static String GetDatabaseDirectory() throws 
MonetDBEmbeddedException {
+if(MonetDBEmbeddedDatabase == null) {
+throw new MonetDBEmbeddedException("The database is not running!");
+}
+return MonetDBEmbeddedDatabase.databaseDirectory;
+}
 
 /**
  * Check if the Silent Flag was set while creating the database.
  *
+ * @throws MonetDBEmbeddedException If the database is not running
  * @return The Silent Flag
  */
-public static boolean IsSilentFlagSet() { return 
MonetDBEmbeddedDatabase.silentFlag; }
+public static boolean IsSilentFlagSet() throws MonetDBEmbeddedException {
+if(MonetDBEmbeddedDatabase == null) {
+throw new MonetDBEmbeddedException("The database is not running!");
+}
+return MonetDBEmbeddedDatabase.silentFlag;
+}
 
 /**
  * Check if the Sequential Flag was set while creating the database.
  *
+ * @throws MonetDBEmbeddedException If the database is not running
  * @return The Sequential Flag
  */
-public static boolean IsSequentialFlagSet() { return 
MonetDBEmbeddedDatabase.sequentialFlag; }
+public static boolean IsSequentialFlagSet() throws 
MonetDBEmbeddedException {
+if(MonetDBEmbeddedDatabase == null) {
+throw new MonetDBEmbeddedException("The database is not running!");
+}
+return MonetDBEmbeddedDatabase.sequentialFlag;
+}
 
 /**
  * Stops the database. All the pending connections will be shut down as 
well.
@@ -95,7 +112,7 @@ public class MonetDBEmbeddedDatabase {
 if(MonetDBEmbeddedDatabase == null) {
 throw new MonetDBEmbeddedException("The database is not running!");
 } else {
-for(MonetDBEmbeddedConnection mdbec : 
MonetDBEmbeddedDatabase.connections) {
+for(MonetDBEmbeddedConnection mdbec : 
MonetDBEmbeddedDatabase.connections.values()) {
 mdbec.closeConnectionImplementation();
 }
 MonetDBEmbeddedDatabase.connections.clear();
@@ -119,7 +136,7 @@ public class MonetDBEmbeddedDatabase {
 
 private final boolean sequentialFlag;
 
-private final Set connections = new HashSet<>();
+private final ConcurrentHashMap<Long, MonetDBEmbeddedConnection> 
connections = new ConcurrentHashMap<>();
 
 private MonetDBEmbeddedDatabase(String dbDirectory, boolean silentFlag, 
boolean sequentialFlag) {
 this.databaseDirectory = dbDirectory;
@@ -134,7 +151,13 @@ public class MonetDBEmbeddedDatabase {
  * @throws MonetDBEmbeddedException If the database is not running or an 
error in the database occurred
  */
 public static MonetDBEmbeddedConnection CreateConnection() throws 
MonetDBEmbeddedException {
-return MonetDBEmbeddedDatabase.createConnectionInternal();
+if(MonetDBEmbeddedDatabase == null) {
+throw new MonetDBEmbeddedException("The database is not running!");
+} else {
+MonetDBEmbeddedConnection con = 
MonetDBEmbeddedDatabase.createConnectionInternal();
+
MonetDBEmbeddedDatabase.connections.put(con.getConnectionPointer(), con);
+return con;
+}
 }
 
 /**
@@ -151,7 +174,7 @@ public class MonetDBEmbeddedDatabase {
  * Removes a connection from this database.
  */
 protected static void RemoveConnection(MonetDBEmbeddedConnection con) {
-MonetDBEmbeddedDatabase.connections.remove(con);
+MonetDBEmbeddedDatabase.connections.remove(con.connectionPointer);
 }
 
 /**
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java 
b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
--- a/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/mapping/MonetDBEmbeddedBlob.java
@@ -19,7 +19,7 @@ import java.util.Collections;
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public class MonetDBEmbeddedBlob implements Serializable, Blob {
+public class MonetDBEmbeddedBlob implements Serializable, Blob, 
Comparable {
 
 /**
  * The BLOB's content as a Java byte array.
@@ -156,4 +156,14 @@ public class MonetDBEmbeddedBlob impleme
 this.checkFreed();
 return new ByteArrayInputStream(Arrays.copyOfRange(this.blob, (int) 
pos, (int) length));
 }
+
+@Override
+public int compareTo(MonetDBEmbeddedBlob o) {
+byte[] first = this.blob, second = o.blob;
+int len = Math.min(first.length, s

monetdb-java: embedded - Consistent iteration limits.

[Pedro Ferreira]

Changeset: 2053a9fc56ca for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=2053a9fc56ca
Modified Files:

src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
Branch: embedded
Log Message:

Consistent iteration limits.


diffs (96 lines):

diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
--- 
a/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
+++ 
b/src/main/java/nl/cwi/monetdb/embedded/tables/IMonetDBTableBaseIterator.java
@@ -8,7 +8,7 @@ package nl.cwi.monetdb.embedded.tables;
 public interface IMonetDBTableBaseIterator {
 
 /**
- * Specify the first row in the table to iterate starting from 1. If a 
lower number is provided, then the iteration
+ * Specify the first row in the table to iterate starting from 0. If a 
lower number is provided, then the iteration
  * will start on the first row.
  *
  * @return The first row in the table to iterate
diff --git a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/MonetDBTable.java
@@ -136,8 +136,8 @@ public class MonetDBTable extends Abstra
 res[0] = res[1];
 res[0] = aux;
 }
-if (res[0] < 1) {
-res[0] = 1;
+if (res[0] < 0) {
+res[0] = 0;
 }
 int numberOfRows = this.getNumberOfRows();
 if (res[1] >= numberOfRows) {
@@ -155,9 +155,9 @@ public class MonetDBTable extends Abstra
  */
 public int iterateTable(IMonetDBTableCursor cursor) throws 
MonetDBEmbeddedException {
 int[] limits = this.prepareIterator(cursor);
-int res = 0, total = limits[1] - limits[0] + 1;
+int res = 0, total = limits[1] - limits[0];
 String query = new StringBuffer("SELECT * FROM 
").append(this.getTableSchema()).append(".").append(this.getTableName())
-.append(" LIMIT ").append(total).append(" OFFSET 
").append(limits[0] - 1).append(";").toString();
+.append(" LIMIT ").append(total).append(" OFFSET 
").append(limits[0]).append(";").toString();
 
 QueryResultSet eqr = this.getConnection().sendQuery(query);
 MonetDBRow[] array = eqr.fetchAllRowValues().getAllRows();
@@ -168,8 +168,9 @@ public class MonetDBTable extends Abstra
 }
 
 RowIterator ri = new RowIterator(this, data, limits[0], limits[1]);
-while(ri.tryContinueIteration()) {
+while(ri.hasMore()) {
 cursor.processNextRow(ri);
+ri.setNextIteration();
 res++;
 }
 return res;
diff --git a/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java 
b/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
--- a/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/tables/RowIterator.java
@@ -22,7 +22,7 @@ public class RowIterator extends Abstrac
 /**
  * The current table row number on the fetched set.
  */
-protected int currentIterationNumber;
+protected int currentIterationNumber = 0;
 
 /**
  * The first row in the table to iterate.
@@ -38,7 +38,6 @@ public class RowIterator extends Abstrac
 super(table, table.getMappings(), rows);
 this.firstIndex = firstIndex;
 this.lastIndex = lastIndex;
-this.currentIterationNumber = -1;
 }
 
 @Override
@@ -101,7 +100,7 @@ public class RowIterator extends Abstrac
  *
  * @return There are more rows to iterate
  */
-public boolean hasMore() { return this.currentIterationNumber + 
this.firstIndex < this.lastIndex; }
+public boolean hasMore() { return this.firstIndex + 
this.currentIterationNumber < this.lastIndex; }
 
 /**
  * Gets a column value as a Java class.
@@ -153,15 +152,7 @@ public class RowIterator extends Abstrac
 }
 
 /**
- * Get the next row in the table if there are more.
- *
- * @return A boolean indicating if there are more rows to fetch
+ * Sets the next value to iterate.
  */
-protected boolean tryContinueIteration() {
-if(this.hasMore()) {
-this.currentIterationNumber++;
-return true;
-}
-return false;
-}
+protected void setNextIteration() { this.currentIterationNumber++; }
 }
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

monetdb-java: embedded - Better handling with concurrent connect...

[Pedro Ferreira]

Changeset: 80bae18085f0 for monetdb-java
URL: http://dev.monetdb.org/hg/monetdb-java?cmd=changeset;node=80bae18085f0
Modified Files:
src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
Branch: embedded
Log Message:

Better handling with concurrent connections.


diffs (111 lines):

diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/AbstractConnectionResult.java
@@ -9,6 +9,7 @@
 package nl.cwi.monetdb.embedded.env;
 
 import java.io.Closeable;
+import java.util.Random;
 
 /**
  * The base class for a pending statement to a connection.
@@ -18,20 +19,38 @@ import java.io.Closeable;
 public abstract class AbstractConnectionResult implements Closeable {
 
 /**
+ * A random instance to generate the result set identifier.
+ */
+private static final Random Randomizer = new Random();
+
+/**
  * The corresponding connection of this result.
  */
 private final MonetDBEmbeddedConnection connection;
 
-protected AbstractConnectionResult(MonetDBEmbeddedConnection connection) { 
this.connection = connection; }
+/**
+ * A long value used to identify this result set.
+ */
+private final long randomIdentifier;
+
+protected AbstractConnectionResult(MonetDBEmbeddedConnection connection) {
+this.connection = connection;
+this.randomIdentifier = Randomizer.nextLong();
+}
 
 /**
- * Get the corresponding connection to this statement result.
+ * Gets the corresponding connection to this statement result.
  *
  * @return A MonetDBEmbeddedConnection instance
  */
 public MonetDBEmbeddedConnection getConnection() { return connection; }
 
-protected long getConnectionPointer() { return 
connection.connectionPointer; }
+/**
+ * Gets a long number randomly generated, used to identify the result set.
+ *
+ * @return A random long identifier
+ */
+protected long getRandomIdentifier() { return randomIdentifier; }
 
 /**
  * To be called by the connection when is closing.
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
@@ -96,7 +96,7 @@ public class MonetDBEmbeddedConnection {
 query += ";";
 }
 UpdateResultSet res = this.sendUpdateInternal(this.connectionPointer, 
query, true);
-results.put(res.getConnectionPointer(), res);
+results.put(res.getRandomIdentifier(), res);
 return res;
 }
 
@@ -123,7 +123,7 @@ public class MonetDBEmbeddedConnection {
 query += ";";
}
 QueryResultSet res = this.sendQueryInternal(this.connectionPointer, 
query, true);
-results.put(res.getConnectionPointer(), res);
+results.put(res.getRandomIdentifier(), res);
 return res;
}
 
@@ -148,7 +148,7 @@ public class MonetDBEmbeddedConnection {
  */
 public MonetDBTable getMonetDBTable(String schemaName, String tableName) 
throws MonetDBEmbeddedException {
 MonetDBTable res = 
this.getMonetDBTableInternal(this.connectionPointer, schemaName, tableName);
-results.put(res.getConnectionPointer(), res);
+results.put(res.getRandomIdentifier(), res);
 return res;
 }
 
@@ -240,7 +240,7 @@ public class MonetDBEmbeddedConnection {
 /**
  * Removes a query result from this connection.
  */
-protected void removeQueryResult(AbstractConnectionResult res) { 
this.results.remove(res.getConnectionPointer()); }
+protected void removeQueryResult(AbstractConnectionResult res) { 
this.results.remove(res.getRandomIdentifier()); }
 
 /**
  * Internal implementation of sendUpdate.
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedException.java
@@ -8,12 +8,14 @@
 
 package nl.cwi.monetdb.embedded.env;
 
+import java.sql.SQLException;
+
 /**
  * The exception fired from embedded methods.
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public class MonetDBEmbeddedException extends Exception {
+public class MonetDBEmbeddedException extends SQLException {
 
 public MonetDBE

monetdb-java: embedded - First changes to the JDBC driver.

[Pedro Ferreira]

ublic interface IEmbeddedConnection {
+
+long getConnectionPointer();
+
+void closeConnectionImplementation();
+}
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedConnection.java
@@ -22,15 +22,15 @@ import java.util.concurrent.ConcurrentHa
  *
  * @author mailto:pedro.ferre...@monetdbsolutions.com;>Pedro 
Ferreira
  */
-public class MonetDBEmbeddedConnection {
+public class MonetDBEmbeddedConnection implements IEmbeddedConnection {
 
-   protected final long connectionPointer;
+   private final long connectionPointer;
 
 private final ConcurrentHashMap<Long, AbstractConnectionResult> results = 
new ConcurrentHashMap<>();
 
protected MonetDBEmbeddedConnection(long connectionPointer) { 
this.connectionPointer = connectionPointer; }
 
-protected long getConnectionPointer() { return connectionPointer; }
+public long getConnectionPointer() { return connectionPointer; }
 
 /**
  * Gets the current schema set on the connection.
@@ -215,7 +215,7 @@ public class MonetDBEmbeddedConnection {
 /**
  * When the database is shuts down, this method is called instead
  */
-protected void closeConnectionImplementation() {
+public void closeConnectionImplementation() {
 for(AbstractConnectionResult res : this.results.values()) {
 res.closeImplementation();
 }
diff --git 
a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java 
b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java
--- a/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java
+++ b/src/main/java/nl/cwi/monetdb/embedded/env/MonetDBEmbeddedDatabase.java
@@ -8,6 +8,8 @@
 
 package nl.cwi.monetdb.embedded.env;
 
+import nl.cwi.monetdb.mcl.embedded.EmbeddedConnection;
+
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -112,7 +114,7 @@ public class MonetDBEmbeddedDatabase {
 if(MonetDBEmbeddedDatabase == null) {
 throw new MonetDBEmbeddedException("The database is not running!");
 } else {
-for(MonetDBEmbeddedConnection mdbec : 
MonetDBEmbeddedDatabase.connections.values()) {
+for(IEmbeddedConnection mdbec : 
MonetDBEmbeddedDatabase.connections.values()) {
 mdbec.closeConnectionImplementation();
 }
 MonetDBEmbeddedDatabase.connections.clear();
@@ -136,7 +138,7 @@ public class MonetDBEmbeddedDatabase {
 
 private final boolean sequentialFlag;
 
-private final ConcurrentHashMap<Long, MonetDBEmbeddedConnection> 
connections = new ConcurrentHashMap<>();
+private final ConcurrentHashMap<Long, IEmbeddedConnection> connections = 
new ConcurrentHashMap<>();
 
 private MonetDBEmbeddedDatabase(String dbDirectory, boolean silentFlag, 
boolean sequentialFlag) {
 this.databaseDirectory = dbDirectory;
@@ -170,11 +172,20 @@ public class MonetDBEmbeddedDatabase {
 return CompletableFuture.supplyAsync(() -> 
this.createConnectionInternal());
 }*/
 
+public static void AddJDBCEmbeddedConnection(EmbeddedConnection con) 
throws MonetDBEmbeddedException {
+if(MonetDBEmbeddedDatabase == null) {
+throw new MonetDBEmbeddedException("The database is not running!");
+} else {
+MonetDBEmbeddedDatabase.createJDBCConnectionInternal(con);
+
MonetDBEmbeddedDatabase.connections.put(con.getConnectionPointer(), con);
+}
+}
+
 /**
  * Removes a connection from this database.
  */
 protected static void RemoveConnection(MonetDBEmbeddedConnection con) {
-MonetDBEmbeddedDatabase.connections.remove(con.connectionPointer);
+MonetDBEmbeddedDatabase.connections.remove(con.getConnectionPointer());
 }
 
 /**
@@ -193,4 +204,9 @@ public class MonetDBEmbeddedDatabase {
  * Internal implementation to create a connection on this database.
  */
 private native MonetDBEmbeddedConnection createConnectionInternal() throws 
MonetDBEmbeddedException;
+
+/**
+ * Internal implementation to create a JDBC embeddded connection on this 
database.
+ */
+private native void createJDBCConnectionInternal(EmbeddedConnection emc) 
throws MonetDBEmbeddedException;
 }
diff --git a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java 
b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java
@@ -44,9 +44,11 @@ import java.util.concurrent.locks.Reentr
 import nl.cwi.monetdb.jdbc.types.INET;
 import nl.cwi.monetdb.jdbc.types.URL;
 import nl.cwi.monet