Changeset: 891059aea396 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=891059aea396
Added Files:
clients/iotclient/tests/__init__.py
clients/iotclient/tests/datatypestests.py
clients/iotclient/tests/main.py
Modified Files:
clients/iotapi/documentation/api_server_arguments.rst
clients/iotapi/src/Settings/filesystem.py
clients/iotapi/src/Settings/iotlogger.py
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/requirements.txt
clients/iotclient/src/Settings/iotlogger.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/main.py
Branch: iot
Log Message:
Added first batch of tests
diffs (truncated from 774 to 300 lines):
diff --git a/clients/iotapi/documentation/api_server_arguments.rst
b/clients/iotapi/documentation/api_server_arguments.rst
--- a/clients/iotapi/documentation/api_server_arguments.rst
+++ b/clients/iotapi/documentation/api_server_arguments.rst
@@ -75,6 +75,6 @@ Name of database to use. By default is :
Help
----
-**- -help**
+**-he - -help**
Display arguments help.
diff --git a/clients/iotapi/src/Settings/filesystem.py
b/clients/iotapi/src/Settings/filesystem.py
--- a/clients/iotapi/src/Settings/filesystem.py
+++ b/clients/iotapi/src/Settings/filesystem.py
@@ -11,11 +11,11 @@ elif sys.platform == "win32":
DEFAULT_FILESYSTEM = os.path.join(os.path.dirname(__file__), os.pardir)
-def init_file_system(new_location=None):
+def init_file_system(filesystem_location):
global Baskets_Location
try:
- Baskets_Location = os.path.join(new_location, "baskets")
+ Baskets_Location = os.path.join(filesystem_location, "baskets")
if not os.path.exists(Baskets_Location):
os.makedirs(Baskets_Location)
except (Exception, OSError) as ex:
diff --git a/clients/iotapi/src/Settings/iotlogger.py
b/clients/iotapi/src/Settings/iotlogger.py
--- a/clients/iotapi/src/Settings/iotlogger.py
+++ b/clients/iotapi/src/Settings/iotlogger.py
@@ -12,13 +12,11 @@ elif sys.platform == "win32":
def init_logging(logging_location):
try:
- Logger.setLevel(logging.DEBUG)
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s
- %(message)s')
logging_path = os.path.dirname(logging_location)
if not os.path.exists(logging_path):
os.makedirs(logging_path)
log_handler = logging.FileHandler(logging_location, mode='a+')
- log_handler.setFormatter(formatter)
+ log_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s -
%(levelname)s - %(message)s'))
Logger.addHandler(log_handler)
except (Exception, OSError) as ex:
print ex
diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -77,7 +77,7 @@ def main():
help='Database listening port (default: 50000)',
metavar='PORT')
parser.add_argument('-d', '--database', nargs='?', default='iotdb',
help='Database name (default: iotdb)')
parser.add_argument('-u', '--user', nargs='?', default='monetdb',
help='Database user (default: monetdb)')
- parser.add_argument('--help', action='store_true', help='Display this
help')
+ parser.add_argument('-he', '--help', action='store_true', help='Display
this help')
try:
args = vars(parser.parse_args())
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -92,6 +92,6 @@ Name of database to use. By default is :
Help
----
-**- -help**
+**-he - -help**
Display arguments help.
diff --git a/clients/iotclient/requirements.txt
b/clients/iotclient/requirements.txt
--- a/clients/iotclient/requirements.txt
+++ b/clients/iotclient/requirements.txt
@@ -1,8 +1,10 @@
+python-dateutil==2.5.3
+fake-factory==0.5.7
Flask-RESTful==0.3.5
IPy==0.83
+iso8601==0.1.11
jsonschema==2.5.1
pymonetdb==0.1.1
-python-dateutil==2.5.3
pytz==2016.4
rfc3987==1.3.6
Sphinx==1.4.1
diff --git a/clients/iotclient/src/Settings/iotlogger.py
b/clients/iotclient/src/Settings/iotlogger.py
--- a/clients/iotclient/src/Settings/iotlogger.py
+++ b/clients/iotclient/src/Settings/iotlogger.py
@@ -12,13 +12,11 @@ elif sys.platform == "win32":
def init_logging(logging_location):
try:
- Logger.setLevel(logging.DEBUG)
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s
- %(message)s')
logging_path = os.path.dirname(logging_location)
if not os.path.exists(logging_path):
os.makedirs(logging_path)
log_handler = logging.FileHandler(logging_location, mode='a+')
- log_handler.setFormatter(formatter)
+ log_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s -
%(levelname)s - %(message)s'))
Logger.addHandler(log_handler)
except (Exception, OSError) as ex:
print ex
diff --git a/clients/iotclient/src/Streams/datatypes.py
b/clients/iotclient/src/Streams/datatypes.py
--- a/clients/iotclient/src/Streams/datatypes.py
+++ b/clients/iotclient/src/Streams/datatypes.py
@@ -1,12 +1,13 @@
import copy
import datetime
+import iso8601
import itertools
import math
import re
import struct
+from abc import ABCMeta, abstractmethod
from dateutil import parser
-from abc import ABCMeta, abstractmethod
from jsonschemas import UUID_REGEX, MAC_ADDRESS_REGEX, TIME_REGEX, IPV4_REGEX
# The null constants might change from system to system due to different CPU's
limits
@@ -548,11 +549,10 @@ class DecimalType(NumberBaseType):
def process_next_value(self, entry, counter, parameters, errors):
self.check_value_precision(entry, 'entry')
- parsed_value = int(entry)
if self._pack_sym != 'Q':
- return parsed_value
+ return int(entry)
else:
- return [parsed_value & INT64_MAX, (parsed_value >> 64) & INT64_MAX]
+ return [int(entry) & INT64_MAX, (int(entry) >> 64) & INT64_MAX]
def pack_parsed_values(self, extracted_values, counter, parameters):
if self._pack_sym == 'Q':
@@ -610,7 +610,7 @@ class BaseDateTimeType(StreamDataType):
pass
def process_next_value(self, entry, counter, parameters, errors):
- if entry == self.get_nullable_constant(): # have to do this trick due
to Python's datetime limitations
+ if entry == self.get_nullable_constant(): # have to do this trick due
to Python datetime limitations
return self.pack_next_value(None, counter, parameters, errors)
parsed = self.parse_entry(entry)
if hasattr(self, '_minimum') and not hasattr(self, '_maximum') and
parsed < self._minimum:
@@ -643,7 +643,7 @@ class DateType(BaseDateTimeType): # Sto
schema[self._column_name]['format'] = 'date'
def parse_entry(self, entry):
- return datetime.datetime.strptime(str(entry), "%Y-%m-%d")
+ return datetime.datetime.strptime(entry, "%Y-%m-%d")
def pack_next_value(self, parsed, counter, parameters, errors):
if parsed is None:
@@ -663,7 +663,6 @@ class TimeType(BaseDateTimeType): # Sto
super(TimeType, self).__init__(**kwargs)
if 'timezone' in kwargs:
self._has_timezone = kwargs['timezone']
- self._data_type = 'timez'
else:
self._has_timezone = True
@@ -672,10 +671,14 @@ class TimeType(BaseDateTimeType): # Sto
schema[self._column_name]['pattern'] = TIME_REGEX
def parse_entry(self, entry):
- parsed_time = datetime.datetime.strptime(str(entry), "%H:%M:%S.%f")
- if not self._has_timezone:
- parsed_time = parsed_time.replace(tzinfo=None)
- return parsed_time
+ parsed = parser.parse(entry)
+ if self._has_timezone:
+ string = parsed.strftime("%z")
+ delta = datetime.timedelta(hours=int(string[1:3]),
minutes=int(string[3:5]))
+ if string[0] == '-':
+ delta = -delta
+ parsed = parsed.replace(tzinfo=None) - delta
+ return parsed
def pack_next_value(self, parsed, counter, parameters, errors):
if parsed is None:
@@ -687,6 +690,15 @@ class TimeType(BaseDateTimeType): # Sto
def pack_parsed_values(self, extracted_values, counter, parameters):
return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter) + 'I',
*extracted_values)
+ def process_sql_parameters(self, array):
+ if self._has_timezone:
+ array[2] = 'time with timezone'
+
+ def to_json_representation(self):
+ json_value = super(TimeType, self).to_json_representation()
+ json_value['has_timezone'] = self._has_timezone
+ return json_value
+
class TimestampType(BaseDateTimeType): # it's represented with the two
integers from time and date
"""Covers: TIMESTAMP"""
@@ -695,7 +707,6 @@ class TimestampType(BaseDateTimeType):
super(TimestampType, self).__init__(**kwargs)
if 'timezone' in kwargs:
self._has_timezone = kwargs['timezone']
- self._data_type = 'timestampz'
else:
self._has_timezone = True
@@ -704,10 +715,14 @@ class TimestampType(BaseDateTimeType):
schema[self._column_name]['format'] = 'date-time'
def parse_entry(self, entry):
- parsed_timestamp = parser.parse(entry)
- if not self._has_timezone:
- parsed_timestamp = parsed_timestamp.replace(tzinfo=None)
- return parsed_timestamp
+ parsed = iso8601.parse_date(entry)
+ if self._has_timezone:
+ string = parsed.strftime("%z")
+ delta = datetime.timedelta(hours=int(string[1:3]),
minutes=int(string[3:5]))
+ if string[0] == '-':
+ delta = -delta
+ parsed = parsed.replace(tzinfo=None) - delta
+ return parsed
def pack_next_value(self, parsed, counter, parameters, errors):
if parsed is None:
@@ -722,3 +737,12 @@ class TimestampType(BaseDateTimeType):
def pack_parsed_values(self, extracted_values, counter, parameters):
concat_array = list(itertools.chain(*extracted_values))
return struct.pack(LITTLE_ENDIAN_ALIGNMENT + str(counter << 1) + 'I',
*concat_array)
+
+ def process_sql_parameters(self, array):
+ if self._has_timezone:
+ array[2] = 'timestamp with timezone'
+
+ def to_json_representation(self):
+ json_value = super(TimestampType, self).to_json_representation()
+ json_value['has_timezone'] = self._has_timezone
+ return json_value
diff --git a/clients/iotclient/src/Streams/jsonschemas.py
b/clients/iotclient/src/Streams/jsonschemas.py
--- a/clients/iotclient/src/Streams/jsonschemas.py
+++ b/clients/iotclient/src/Streams/jsonschemas.py
@@ -1,7 +1,7 @@
UUID_REGEX = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
IPV4_REGEX =
"^(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}(?:\/[0-2]\d|\/3[0-2])?$"
MAC_ADDRESS_REGEX = "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"
-TIME_REGEX = "^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\.\d{3}$"
+TIME_REGEX =
"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\.\d{3}[\+-]([01]\d|2[0-3]):[0-5]\d$"
TIMED_FLUSH_IDENTIFIER = "time"
TUPLE_FLUSH_IDENTIFIER = "tuple"
diff --git a/clients/iotclient/src/Streams/streams.py
b/clients/iotclient/src/Streams/streams.py
--- a/clients/iotclient/src/Streams/streams.py
+++ b/clients/iotclient/src/Streams/streams.py
@@ -86,7 +86,7 @@ class BaseIOTStream(object):
# WARNING DELETE this code afterwards is for debugging purposes for now
basket_counter_file_pointer =
open(os.path.join(self._current_base_path, BASKETS_COUNT_FILE), "w+b")
- basket_counter_file_pointer.write(struct.pack(LITTLE_ENDIAN_ALIGNMENT
+ "i",0))
+ basket_counter_file_pointer.write(struct.pack(LITTLE_ENDIAN_ALIGNMENT
+ "i", 0))
basket_counter_file_pointer.flush()
basket_counter_file_pointer.close()
# END WARNING
diff --git a/clients/iotclient/src/main.py b/clients/iotclient/src/main.py
--- a/clients/iotclient/src/main.py
+++ b/clients/iotclient/src/main.py
@@ -94,7 +94,7 @@ def main():
help='Database listening port (default: 50000)',
metavar='PORT')
parser.add_argument('-d', '--database', nargs='?', default='iotdb',
help='Database name (default: iotdb)')
parser.add_argument('-u', '--user', nargs='?', default='monetdb',
help='Database user (default: monetdb)')
- parser.add_argument('--help', action='store_true', help='Display this
help')
+ parser.add_argument('-he', '--help', action='store_true', help='Display
this help')
try:
args = vars(parser.parse_args())
@@ -110,7 +110,7 @@ def main():
con_password = getpass.getpass(prompt='Insert password for user ' +
args['user'] + ':')
init_logging(args['log']) # init logging context
- subprocess = Process(target=start_process, args=(args['filesystem'],
args['useidentifier'], args['name'],
+ subprocess = Process(target=start_process, args=(args['filesystem'],
args['identifier'], args['name'],
args['ihost'],
args['iport'], args['ahost'], args['aport'],
args['host'],
args['port'], args['user'], con_password,
args['database']))
diff --git a/clients/iotclient/tests/__init__.py
b/clients/iotclient/tests/__init__.py
new file mode 100644
diff --git a/clients/iotclient/tests/datatypestests.py
b/clients/iotclient/tests/datatypestests.py
new file mode 100644
--- /dev/null
+++ b/clients/iotclient/tests/datatypestests.py
@@ -0,0 +1,398 @@
+import os
+import random
+import uuid
+import unittest
+
+from abc import ABCMeta, abstractmethod
+from faker import Factory
+from pytz import timezone
+
+
+def load_src(name, fpath):
+ import os
+ import imp
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
