Changeset: 23ae9129b7da for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=23ae9129b7da
Modified Files:
monetdb5/extras/pyapi/pyapi.c
sql/backends/monet5/Tests/pyapi00.sql
sql/backends/monet5/Tests/pyapi01.sql
sql/backends/monet5/Tests/pyapi02.sql
sql/backends/monet5/Tests/pyapi03.sql
sql/backends/monet5/Tests/pyapi04.sql
sql/backends/monet5/Tests/pyapi05.sql
sql/backends/monet5/Tests/pyapi06.sql
sql/backends/monet5/Tests/pyapi07.sql
sql/backends/monet5/Tests/pyapi08.sql
sql/backends/monet5/Tests/pyapi10.sql
sql/backends/monet5/Tests/pyapi10.stable.err
sql/backends/monet5/Tests/pyapi11.sql
sql/backends/monet5/Tests/pyapi12.sql
sql/backends/monet5/Tests/pyapi13.sql
sql/backends/monet5/Tests/pyapi16.sql
sql/backends/monet5/Tests/pyapi17.sql
sql/backends/monet5/Tests/pyapi18.sql
sql/backends/monet5/Tests/pyapi19.sql
sql/backends/monet5/Tests/pyapi20.sql
sql/backends/monet5/Tests/pyapi21.sql
sql/backends/monet5/Tests/pyapi22.sql
sql/backends/monet5/Tests/pyapi23.sql
sql/backends/monet5/Tests/pyapi24.sql
sql/backends/monet5/Tests/pyapi24.stable.out
Branch: pyapi
Log Message:
Add comments to pyapi SQL tests for clarity.
diffs (truncated from 368 to 300 lines):
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -1051,7 +1051,6 @@ returnvalues:
}
strcpy(error_mem, msg);
-
exit(1);
}
#endif
diff --git a/sql/backends/monet5/Tests/pyapi00.sql
b/sql/backends/monet5/Tests/pyapi00.sql
--- a/sql/backends/monet5/Tests/pyapi00.sql
+++ b/sql/backends/monet5/Tests/pyapi00.sql
@@ -1,11 +1,11 @@
+# Basic MonetDB/Python test
+# We simply return a list of the values 1-10 and store it in a table
START TRANSACTION;
-
CREATE FUNCTION pyapi00() returns table (d integer)
language P
{
- x = range(1,11)
- return(x)
+ return(range(1,11))
};
SELECT * FROM pyapi00() AS R WHERE d > 5;
diff --git a/sql/backends/monet5/Tests/pyapi01.sql
b/sql/backends/monet5/Tests/pyapi01.sql
--- a/sql/backends/monet5/Tests/pyapi01.sql
+++ b/sql/backends/monet5/Tests/pyapi01.sql
@@ -1,3 +1,4 @@
+# Create a table with multiple columns with MonetDB/Python
START TRANSACTION;
CREATE FUNCTION pyapi01(i integer) returns table (i integer, d double)
diff --git a/sql/backends/monet5/Tests/pyapi02.sql
b/sql/backends/monet5/Tests/pyapi02.sql
--- a/sql/backends/monet5/Tests/pyapi02.sql
+++ b/sql/backends/monet5/Tests/pyapi02.sql
@@ -1,3 +1,4 @@
+# Use MonetDB/Python for some simple SELECT() operations
START TRANSACTION;
CREATE TABLE rval(i integer,j integer);
diff --git a/sql/backends/monet5/Tests/pyapi03.sql
b/sql/backends/monet5/Tests/pyapi03.sql
--- a/sql/backends/monet5/Tests/pyapi03.sql
+++ b/sql/backends/monet5/Tests/pyapi03.sql
@@ -1,3 +1,4 @@
+# Use MonetDB/Python to filter data by creating a simple filter that is
identical to 'i > z' in SQL
START TRANSACTION;
CREATE TABLE rval(i integer);
diff --git a/sql/backends/monet5/Tests/pyapi04.sql
b/sql/backends/monet5/Tests/pyapi04.sql
--- a/sql/backends/monet5/Tests/pyapi04.sql
+++ b/sql/backends/monet5/Tests/pyapi04.sql
@@ -1,3 +1,4 @@
+# Return a couple of boolean values with no parameter provided (i.e. return a
constant)
START TRANSACTION;
CREATE TABLE dval(i integer);
diff --git a/sql/backends/monet5/Tests/pyapi05.sql
b/sql/backends/monet5/Tests/pyapi05.sql
--- a/sql/backends/monet5/Tests/pyapi05.sql
+++ b/sql/backends/monet5/Tests/pyapi05.sql
@@ -1,3 +1,4 @@
+# Do a more complex operation (clustering using scipy)
START TRANSACTION;
CREATE TABLE xdata (x float);
diff --git a/sql/backends/monet5/Tests/pyapi06.sql
b/sql/backends/monet5/Tests/pyapi06.sql
--- a/sql/backends/monet5/Tests/pyapi06.sql
+++ b/sql/backends/monet5/Tests/pyapi06.sql
@@ -1,3 +1,6 @@
+# Test aggregations with the hidden variable 'aggr_group'.
+# Note that if we do an aggregation without GROUP BY the variable 'aggr_group'
does not exist.
+# To handle this case, we have to check if aggr_group is in the locals.
START TRANSACTION;
CREATE TABLE rval(groupcol integer,datacol integer);
diff --git a/sql/backends/monet5/Tests/pyapi07.sql
b/sql/backends/monet5/Tests/pyapi07.sql
--- a/sql/backends/monet5/Tests/pyapi07.sql
+++ b/sql/backends/monet5/Tests/pyapi07.sql
@@ -1,3 +1,5 @@
+# A more complex aggregation, perform a quantile using MonetDB/Python
+# Also perform two aggregations in the same SELECT() statement, which could
potentially fail
START TRANSACTION;
CREATE TABLE rval(val double);
diff --git a/sql/backends/monet5/Tests/pyapi08.sql
b/sql/backends/monet5/Tests/pyapi08.sql
--- a/sql/backends/monet5/Tests/pyapi08.sql
+++ b/sql/backends/monet5/Tests/pyapi08.sql
@@ -1,3 +1,7 @@
+# This test example is from Hannes Muehleisen
+# From a demo he gave about checking when planes are close together to detect
'close calls' by comparing plane coordinates
+ # Nice testcase as a reasonably complex function with multiple different
parameters/return values
+
start transaction;
CREATE TABLE "streams" (
diff --git a/sql/backends/monet5/Tests/pyapi10.sql
b/sql/backends/monet5/Tests/pyapi10.sql
--- a/sql/backends/monet5/Tests/pyapi10.sql
+++ b/sql/backends/monet5/Tests/pyapi10.sql
@@ -1,4 +1,8 @@
-# Test multiprocessing
+# Basic multiprocessing test using PYTHON_MAP
+# We also test if error codes work correctly when the child process fails
+# Because the error occurs in a child process separate from the main process
when multiprocessing is enabled we have to explicity send the error message
back to the main process
+# This could go wrong, so we test it
+
START TRANSACTION;
CREATE FUNCTION pyapi10_random_table(entries integer) returns table (i
integer, j integer)
@@ -93,3 +97,59 @@ language PYTHON_MAP
SELECT AVG(pyapi10_indentation_error(i,j)) FROM
pyapi10_random_table_nulls(5000);
ROLLBACK;
+
+# This function is incorrect on purpose
+CREATE FUNCTION pyapi10_indentation_error(i integer,j integer) returns integer
+language PYTHON_MAP
+{ a = 1;
+ b = 2
+ c = 3
+ d = 4
+ e = 5
+ f = 6
+ g = 7
+ h = 8
+ i = 9
+ j = 10
+ k = 11
+ l = 12
+ m = 13
+ n = 14
+ return a+b+c+d+e+f+g+h
+};
+
+
+# Let's try a runtime exception
+START TRANSACTION;
+
+CREATE FUNCTION pyapi10_random_table_nulls(entries integer) returns table (i
integer, j integer)
+language P
+{
+ import random
+ random.seed(123)
+ results = [numpy.ma.masked_array(numpy.zeros(entries), 0),
numpy.ma.masked_array(numpy.zeros(entries), 0)]
+ for i in range(0,entries):
+ for j in range(0,2):
+ results[j][i] = random.randint(0,100)
+ if results[j][i] < 50:
+ results[j].mask[i] = True
+ return(results)
+};
+
+# This function is incorrect on purpose
+CREATE FUNCTION pyapi10_runtime_exception(i integer,j integer) returns integer
+language PYTHON_MAP
+{
+ return_val = i * j;
+ myint = 5
+ mydivision = 0
+ # comments
+ hello = myint / mydivision
+ # comments
+ return return_val
+};
+
+# Test errors during multiprocessing
+SELECT AVG(pyapi10_runtime_exception(i,j)) FROM
pyapi10_random_table_nulls(5000);
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyapi10.stable.err
b/sql/backends/monet5/Tests/pyapi10.stable.err
--- a/sql/backends/monet5/Tests/pyapi10.stable.err
+++ b/sql/backends/monet5/Tests/pyapi10.stable.err
@@ -31,7 +31,7 @@ stderr of test 'pyapi10` in directory 's
# 15:32:31 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-27693" "--port=33372"
# 15:32:31 >
-MAPI = (monetdb) /var/tmp/mtest-15243/.s.monetdb.38589
+MAPI = (monetdb) /var/tmp/mtest-24315/.s.monetdb.30457
QUERY = SELECT AVG(pyapi10_error(i,j)) FROM pyapi10_random_table_nulls(5000);
ERROR = !MALException:pyapi.eval:Could not parse Python code
!
@@ -39,7 +39,7 @@ ERROR = !MALException:pyapi.eval:Could n
!> 2. return(i*j
! 3.
!invalid syntax (<string>, line 2)
-MAPI = (monetdb) /var/tmp/mtest-15243/.s.monetdb.38589
+MAPI = (monetdb) /var/tmp/mtest-24315/.s.monetdb.30457
QUERY = SELECT AVG(pyapi10_indentation_error(i,j)) FROM
pyapi10_random_table_nulls(5000);
ERROR = !MALException:pyapi.eval:Could not parse Python code
! 7. f = 6
@@ -48,6 +48,15 @@ ERROR = !MALException:pyapi.eval:Could n
! 10. i = 9
! 11. j = 10
!unexpected indent (<string>, line 9)
+MAPI = (monetdb) /var/tmp/mtest-24315/.s.monetdb.30457
+QUERY = SELECT AVG(pyapi10_runtime_exception(i,j)) FROM
pyapi10_random_table_nulls(5000);
+ERROR = !MALException:pyapi.eval:Python exception
+ ! 4. mydivision = 0
+ ! 5. # comments
+ !> 6. hello = myint / mydivision
+ ! 7. # comments
+ ! 8. return return_val
+ !integer division or modulo by zero
# 15:32:31 >
# 15:32:31 > "Done."
diff --git a/sql/backends/monet5/Tests/pyapi11.sql
b/sql/backends/monet5/Tests/pyapi11.sql
--- a/sql/backends/monet5/Tests/pyapi11.sql
+++ b/sql/backends/monet5/Tests/pyapi11.sql
@@ -1,4 +1,5 @@
# Test nested python calls
+# Basically map/reduce type stuff where we make one function execute in
parallel and then pass the data to a blocking operation to do something that
can't be parallelized
START TRANSACTION;
diff --git a/sql/backends/monet5/Tests/pyapi12.sql
b/sql/backends/monet5/Tests/pyapi12.sql
--- a/sql/backends/monet5/Tests/pyapi12.sql
+++ b/sql/backends/monet5/Tests/pyapi12.sql
@@ -1,3 +1,7 @@
+# Some additional tests for multiprocessing
+# Multiprocessing in the WHERE clause can fail if the SEQBASE of the BATs are
not set correctly because the BATs are split up
+# Returning a numpy.object array can fail because numpy.object is essentially
an array of pointers, so we if copy just the pointers into shared memory/mmap
it breaks
+# We fix this by internally converting the data back into a 'normal' Numpy
array of type STRING or INT or whatever (depending on the objects)
START TRANSACTION;
CREATE TABLE rval(i integer);
diff --git a/sql/backends/monet5/Tests/pyapi13.sql
b/sql/backends/monet5/Tests/pyapi13.sql
--- a/sql/backends/monet5/Tests/pyapi13.sql
+++ b/sql/backends/monet5/Tests/pyapi13.sql
@@ -1,4 +1,6 @@
-# Test whitespace
+# Test various whitespace configurations
+# Including tests for multiline strings (strings starting with """)
+
START TRANSACTION;
CREATE FUNCTION pyapi13_random_table_nulls(entries integer) returns table (i
integer, j integer)
diff --git a/sql/backends/monet5/Tests/pyapi16.sql
b/sql/backends/monet5/Tests/pyapi16.sql
--- a/sql/backends/monet5/Tests/pyapi16.sql
+++ b/sql/backends/monet5/Tests/pyapi16.sql
@@ -1,3 +1,4 @@
+# Test the special _columns and _column_types parameters that store
information of the columns
START TRANSACTION;
CREATE TABLE vals(a STRING, b STRING, c STRING, d INTEGER);
diff --git a/sql/backends/monet5/Tests/pyapi17.sql
b/sql/backends/monet5/Tests/pyapi17.sql
--- a/sql/backends/monet5/Tests/pyapi17.sql
+++ b/sql/backends/monet5/Tests/pyapi17.sql
@@ -1,6 +1,10 @@
+# Return dictionary testing
+# If we return a TABLE we can return a (key,value) dictionary with the key
representing the column name instead of returning a table
+# We check if all the columns are properly represented, and if there are too
many columns in the dictionary
+# And test if the columns are properly mapped to the correct columns.
+
START TRANSACTION;
-# Return dictionary testing
# Standard case
CREATE FUNCTION pyapi17() returns TABLE (a STRING, b STRING, c INTEGER, d
BOOLEAN)
diff --git a/sql/backends/monet5/Tests/pyapi18.sql
b/sql/backends/monet5/Tests/pyapi18.sql
--- a/sql/backends/monet5/Tests/pyapi18.sql
+++ b/sql/backends/monet5/Tests/pyapi18.sql
@@ -1,3 +1,4 @@
+# Test the hidden _values dictionary that is used by the user to store Python
objects to be used by later python functions
START TRANSACTION;
CREATE TABLE vals(i INTEGER);
diff --git a/sql/backends/monet5/Tests/pyapi19.sql
b/sql/backends/monet5/Tests/pyapi19.sql
--- a/sql/backends/monet5/Tests/pyapi19.sql
+++ b/sql/backends/monet5/Tests/pyapi19.sql
@@ -1,3 +1,7 @@
+# Test storing input variables in the dictionary directly
+# This breaks because we do zero copy of numeric columns
+# Meaning you are storing a NumPy array that points to a non-persistent BAT,
which might be deleted later
+# So if you then try to access the data it breaks
START TRANSACTION;
CREATE TABLE vals(i INTEGER);
diff --git a/sql/backends/monet5/Tests/pyapi20.sql
b/sql/backends/monet5/Tests/pyapi20.sql
--- a/sql/backends/monet5/Tests/pyapi20.sql
+++ b/sql/backends/monet5/Tests/pyapi20.sql
@@ -1,4 +1,5 @@
-
+# Test error reporting on exceptions, this is different from parse errors
because we get the line number by checking the stack trace
+# We do want to tell the user exactly which line broke
START TRANSACTION;
CREATE TABLE rval(i integer);
diff --git a/sql/backends/monet5/Tests/pyapi21.sql
b/sql/backends/monet5/Tests/pyapi21.sql
--- a/sql/backends/monet5/Tests/pyapi21.sql
+++ b/sql/backends/monet5/Tests/pyapi21.sql
@@ -1,4 +1,4 @@
-
+# Test storing Python objects in the database using the cPickle module to
convert the objects to encoded strings
START TRANSACTION;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
