This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 3f5e701e84cdfb7dc386266b337aded0754a117c
Author: baunsgaard <baunsga...@tugraz.at>
AuthorDate: Mon Nov 15 19:12:24 2021 +0100
[MINOR] Cleanup and update python tests
---
src/main/python/systemds/operator/nodes/list.py | 2 +-
.../python/systemds/operator/nodes/list_access.py | 5 +--
src/main/python/tests/algorithms/test_kmeans.py | 2 +-
.../test_context_creation.py} | 40 +++++++++---------
src/main/python/tests/frame/test_hyperband.py | 47 ++++++++++------------
src/main/python/tests/frame/test_write_read.py | 5 +--
src/main/python/tests/lineage/test_lineagetrace.py | 44 +++++++++-----------
.../test_print.py => list/test_list_readwrite.py} | 30 ++++++++++----
src/main/python/tests/matrix/test_cholesky.py | 20 +++++----
src/main/python/tests/matrix/test_order.py | 24 +++++++----
src/main/python/tests/matrix/test_print.py | 5 +++
src/main/python/tests/script/test_dml_script.py | 11 +++--
12 files changed, 131 insertions(+), 104 deletions(-)
diff --git a/src/main/python/systemds/operator/nodes/list.py
b/src/main/python/systemds/operator/nodes/list.py
index 6f5bfb1..6ad69ca 100644
--- a/src/main/python/systemds/operator/nodes/list.py
+++ b/src/main/python/systemds/operator/nodes/list.py
@@ -80,7 +80,7 @@ class List(OperationNode):
unnamed_input_vars, named_input_vars)
return f'{var_name}={self.operation}({inputs_comma_sep});'
- def compute(self, verbose: bool = False, lineage: bool = False) ->
Union[np.array]:
+ def compute(self, verbose: bool = False, lineage: bool = False) ->
np.array:
return super().compute(verbose, lineage)
def __str__(self):
diff --git a/src/main/python/systemds/operator/nodes/list_access.py
b/src/main/python/systemds/operator/nodes/list_access.py
index a954f9c..1708206 100644
--- a/src/main/python/systemds/operator/nodes/list_access.py
+++ b/src/main/python/systemds/operator/nodes/list_access.py
@@ -21,10 +21,7 @@
__all__ = ["ListAccess"]
-from typing import Dict, Iterable, Sequence, Tuple, Union
-
-import numpy as np
-from py4j.java_gateway import JavaObject
+from typing import Dict, Sequence
from systemds.operator import Frame, Matrix, OperationNode, Scalar
from systemds.script_building.dag import OutputType
diff --git a/src/main/python/tests/algorithms/test_kmeans.py
b/src/main/python/tests/algorithms/test_kmeans.py
index 328cdd4..6369749 100644
--- a/src/main/python/tests/algorithms/test_kmeans.py
+++ b/src/main/python/tests/algorithms/test_kmeans.py
@@ -82,7 +82,7 @@ class TestKMeans(unittest.TestCase):
self.assertTrue(len(corners) == 4)
- def generate_matrices_for_k_means(self, dims: (int, int), seed: int =
1234):
+ def generate_matrices_for_k_means(self, dims, seed: int = 1234):
np.random.seed(seed)
mu, sigma = 0, 0.1
s = np.random.normal(mu, sigma, dims[0] * dims[1])
diff --git a/src/main/python/tests/matrix/test_print.py
b/src/main/python/tests/basics/test_context_creation.py
similarity index 56%
copy from src/main/python/tests/matrix/test_print.py
copy to src/main/python/tests/basics/test_context_creation.py
index b7231e7..1a70deb 100644
--- a/src/main/python/tests/matrix/test_print.py
+++ b/src/main/python/tests/basics/test_context_creation.py
@@ -21,29 +21,31 @@
import unittest
-import numpy as np
from systemds.context import SystemDSContext
-class TestPrint(unittest.TestCase):
+class TestContextCreation(unittest.TestCase):
- sds: SystemDSContext = None
+ def test_same_port(self):
+ # Same port should graciously change port
+ sds1 = SystemDSContext(port=9415)
+ sds2 = SystemDSContext(port=9415)
+ sds1.close()
+ sds2.close()
- @classmethod
- def setUpClass(cls):
- cls.sds = SystemDSContext()
+ def test_create_10_contexts(self):
+ # Creating multiple contexts and closing them should be no problem.
+ for _ in range(0, 10):
+ SystemDSContext().close()
- @classmethod
- def tearDownClass(cls):
- cls.sds.close()
+ def test_create_multiple_context(self):
+ # Creating multiple contexts in sequence but open at the same time is
okay.
+ a = SystemDSContext()
+ b = SystemDSContext()
+ c = SystemDSContext()
+ d = SystemDSContext()
- def test_print_01(self):
- self.sds.from_numpy(np.array([1])).to_string().print().compute()
- self.assertEqual(1,float(self.sds.get_stdout()[0]))
-
- def test_print_02(self):
- self.sds.scalar(1).print().compute()
- self.assertEqual(1,float(self.sds.get_stdout()[0]))
-
-if __name__ == "__main__":
- unittest.main(exit=False)
+ a.close()
+ b.close()
+ c.close()
+ d.close()
diff --git a/src/main/python/tests/frame/test_hyperband.py
b/src/main/python/tests/frame/test_hyperband.py
index 78a12c7..d8cd958 100644
--- a/src/main/python/tests/frame/test_hyperband.py
+++ b/src/main/python/tests/frame/test_hyperband.py
@@ -54,32 +54,29 @@ class TestHyperband(unittest.TestCase):
pass
def test_hyperband(self):
- if "SYSTEMDS_ROOT" in os.environ:
- x_train = self.sds.from_numpy(self.X_train)
- y_train = self.sds.from_numpy(self.y_train)
- x_val = self.sds.from_numpy(self.X_val)
- y_val = self.sds.from_numpy(self.y_val)
- paramRanges = self.sds.from_numpy(self.param_ranges)
- params = self.params
- [best_weights_mat, opt_hyper_params_df] = hyperband(
- X_train=x_train,
- y_train=y_train,
- X_val=x_val,
- y_val=y_val,
- params=params,
- paramRanges=paramRanges,
- ).compute()
- self.assertTrue(isinstance(best_weights_mat, np.ndarray))
- self.assertTrue(best_weights_mat.shape[0] == self.X_train.shape[1])
- self.assertTrue(best_weights_mat.shape[1] == self.y_train.shape[1])
+ x_train = self.sds.from_numpy(self.X_train)
+ y_train = self.sds.from_numpy(self.y_train)
+ x_val = self.sds.from_numpy(self.X_val)
+ y_val = self.sds.from_numpy(self.y_val)
+ paramRanges = self.sds.from_numpy(self.param_ranges)
+ params = self.params
+ [best_weights_mat, opt_hyper_params_df] = hyperband(
+ X_train=x_train,
+ y_train=y_train,
+ X_val=x_val,
+ y_val=y_val,
+ params=params,
+ paramRanges=paramRanges,
+ ).compute()
+ self.assertTrue(isinstance(best_weights_mat, np.ndarray))
+ self.assertTrue(best_weights_mat.shape[0] == self.X_train.shape[1])
+ self.assertTrue(best_weights_mat.shape[1] == self.y_train.shape[1])
- self.assertTrue(isinstance(opt_hyper_params_df, pd.DataFrame))
- self.assertTrue(opt_hyper_params_df.shape[1] == 1)
- for i, hyper_param in
enumerate(opt_hyper_params_df.values.flatten().tolist()):
- self.assertTrue(
- self.min_max_params[i][0] <= hyper_param <=
self.min_max_params[i][1])
- else:
- print("to enable hyperband tests, set SYSTEMDS_ROOT")
+ self.assertTrue(isinstance(opt_hyper_params_df, pd.DataFrame))
+ self.assertTrue(opt_hyper_params_df.shape[1] == 1)
+ for i, hyper_param in
enumerate(opt_hyper_params_df.values.flatten().tolist()):
+ self.assertTrue(
+ self.min_max_params[i][0] <= hyper_param <=
self.min_max_params[i][1])
if __name__ == "__main__":
diff --git a/src/main/python/tests/frame/test_write_read.py
b/src/main/python/tests/frame/test_write_read.py
index ffa417e..cbbad68 100644
--- a/src/main/python/tests/frame/test_write_read.py
+++ b/src/main/python/tests/frame/test_write_read.py
@@ -19,9 +19,7 @@
#
# -------------------------------------------------------------
-import os
import shutil
-import sys
import unittest
import pandas as pd
@@ -63,7 +61,8 @@ class TestWriteRead(unittest.TestCase):
def test_write_read_csv(self):
frame = self.sds.from_pandas(self.df)
frame.write(self.temp_dir + "02", header=True, format="csv").compute()
- NX = self.sds.read(self.temp_dir + "02", data_type="frame",
format="csv")
+ NX = self.sds.read(self.temp_dir + "02",
+ data_type="frame", format="csv")
result_df = NX.compute()
self.assertTrue(isinstance(result_df, pd.DataFrame))
self.assertTrue(self.df.equals(result_df))
diff --git a/src/main/python/tests/lineage/test_lineagetrace.py
b/src/main/python/tests/lineage/test_lineagetrace.py
index 9f7528b..a223797 100644
--- a/src/main/python/tests/lineage/test_lineagetrace.py
+++ b/src/main/python/tests/lineage/test_lineagetrace.py
@@ -21,11 +21,9 @@
import os
import shutil
-import sys
import unittest
from systemds.context import SystemDSContext
-from systemds.utils.helpers import get_module_dir
os.environ['SYSDS_QUIET'] = "1"
@@ -48,29 +46,27 @@ class TestLineageTrace(unittest.TestCase):
def tearDown(self):
shutil.rmtree(temp_dir, ignore_errors=True)
+ @unittest.skipIf("SYSTEMDS_ROOT" not in os.environ, "The test is skipped
if SYSTEMDS_ROOT is not set, this is required for this tests since it use the
bin/systemds file to execute a reference")
def test_compare_trace1(self): # test getLineageTrace() on an intermediate
- if "SYSTEMDS_ROOT" in os.environ:
- m = self.sds.full((10, 10), 1)
- m_res = m + m
-
- python_trace = [x.strip().split("°")
- for x in m_res.get_lineage_trace().split("\n")]
-
- dml_script = (
- "x = matrix(1, rows=10, cols=10);\n"
- "y = x + x;\n"
- "print(lineage(y));\n"
- )
-
- sysds_trace = create_execute_and_trace_dml(dml_script, "trace1")
-
- # It is not garantied, that the two lists 100% align to be the
same.
- # Therefore for now, we only compare if the command is the same,
in same order.
- python_trace_commands = [x[:1] for x in python_trace]
- dml_script_commands = [x[:1] for x in sysds_trace]
- self.assertEqual(python_trace_commands[0], dml_script_commands[0])
- else:
- print("to enable lineage tests, set SYSTEMDS_ROOT")
+ m = self.sds.full((10, 10), 1)
+ m_res = m + m
+
+ python_trace = [x.strip().split("°")
+ for x in m_res.get_lineage_trace().split("\n")]
+
+ dml_script = (
+ "x = matrix(1, rows=10, cols=10);\n"
+ "y = x + x;\n"
+ "print(lineage(y));\n"
+ )
+
+ sysds_trace = create_execute_and_trace_dml(dml_script, "trace1")
+
+ # It is not garantied, that the two lists 100% align to be the same.
+ # Therefore for now, we only compare if the command is the same, in
same order.
+ python_trace_commands = [x[:1] for x in python_trace]
+ dml_script_commands = [x[:1] for x in sysds_trace]
+ self.assertEqual(python_trace_commands[0], dml_script_commands[0])
# TODO add more tests cases.
diff --git a/src/main/python/tests/matrix/test_print.py
b/src/main/python/tests/list/test_list_readwrite.py
similarity index 58%
copy from src/main/python/tests/matrix/test_print.py
copy to src/main/python/tests/list/test_list_readwrite.py
index b7231e7..0ec0cb5 100644
--- a/src/main/python/tests/matrix/test_print.py
+++ b/src/main/python/tests/list/test_list_readwrite.py
@@ -19,15 +19,17 @@
#
# -------------------------------------------------------------
+import shutil
import unittest
import numpy as np
from systemds.context import SystemDSContext
-class TestPrint(unittest.TestCase):
+class TestListOperations(unittest.TestCase):
sds: SystemDSContext = None
+ temp_dir: str = "tests/list/tmp/readwrite/"
@classmethod
def setUpClass(cls):
@@ -36,14 +38,26 @@ class TestPrint(unittest.TestCase):
@classmethod
def tearDownClass(cls):
cls.sds.close()
+ shutil.rmtree(cls.temp_dir)
+
+ def test_write_followed_by_read(self):
+ ''' Test write and read of lists variables in python.
+ Since we do not support serializing a list (from java to python) yet we
+ read and compute each list element when reading again
+ '''
+ m1 = np.array([[1., 2., 3.]])
+ m1p = self.sds.from_numpy(m1)
+ m2 = np.array([[4., 5., 6.]])
+ m2p = self.sds.from_numpy(m2)
+ list_obj = self.sds.array(m1p, m2p)
+
+ path = self.temp_dir + "01"
+ list_obj.write(path).compute()
+ ret_m1 = self.sds.read(path)[1].as_matrix().compute()
+ ret_m2 = self.sds.read(path)[2].as_matrix().compute()
+ self.assertTrue(np.allclose(m1, ret_m1))
+ self.assertTrue(np.allclose(m2, ret_m2))
- def test_print_01(self):
- self.sds.from_numpy(np.array([1])).to_string().print().compute()
- self.assertEqual(1,float(self.sds.get_stdout()[0]))
-
- def test_print_02(self):
- self.sds.scalar(1).print().compute()
- self.assertEqual(1,float(self.sds.get_stdout()[0]))
if __name__ == "__main__":
unittest.main(exit=False)
diff --git a/src/main/python/tests/matrix/test_cholesky.py
b/src/main/python/tests/matrix/test_cholesky.py
index 64772ed..d6ba5ba 100644
--- a/src/main/python/tests/matrix/test_cholesky.py
+++ b/src/main/python/tests/matrix/test_cholesky.py
@@ -30,6 +30,7 @@ A = np.random.rand(shape, shape)
# set A = MM^T and A is a positive definite matrix
A = np.matmul(A, A.transpose())
+
class TestCholesky(unittest.TestCase):
sds: SystemDSContext = None
@@ -43,7 +44,7 @@ class TestCholesky(unittest.TestCase):
cls.sds.close()
-class TestCholesky_0(TestCholesky):
+class TestCholeskyValid(TestCholesky):
def test_basic1(self):
L = self.sds.from_numpy(A).cholesky().compute()
@@ -54,24 +55,27 @@ class TestCholesky_0(TestCholesky):
# L * L.H = A
self.assertTrue(np.allclose(A, np.dot(L, L.T.conj())))
-class TestCholesky_1(TestCholesky):
+
+class TestCholeskyInvalid_1(TestCholesky):
def test_pos_def(self):
m1 = -np.random.rand(shape, shape)
- with self.assertRaises(RuntimeError) as context:
+ with self.assertRaises(Exception):
self.sds.from_numpy(m1).cholesky().compute()
-
-class TestCholesky_2(TestCholesky):
+
+
+class TestCholeskyInvalid_2(TestCholesky):
def test_symmetric_matrix(self):
m2 = np.asarray([[4, 9], [1, 4]])
np.linalg.cholesky(m2)
- with self.assertRaises(RuntimeError) as context:
+ with self.assertRaises(Exception):
self.sds.from_numpy(m2).cholesky().compute()
-class TestCholesky_3(TestCholesky):
+
+class TestCholeskyInvalid_3(TestCholesky):
def test_asymetric_dim(self):
m3 = np.random.rand(shape, shape + 1)
- with self.assertRaises(RuntimeError) as context:
+ with self.assertRaises(Exception):
self.sds.from_numpy(m3).cholesky().compute()
diff --git a/src/main/python/tests/matrix/test_order.py
b/src/main/python/tests/matrix/test_order.py
index 24878e1..cd88ac4 100644
--- a/src/main/python/tests/matrix/test_order.py
+++ b/src/main/python/tests/matrix/test_order.py
@@ -19,8 +19,8 @@
#
# -------------------------------------------------------------
-import unittest
import random
+import unittest
import numpy as np
from systemds.context import SystemDSContext
@@ -33,7 +33,8 @@ mx = np.random.rand(1, shape[1])
my = np.random.rand(shape[0], 1)
by = random.randrange(1, np.size(m, 1)+1)
-class TestOrder(unittest.TestCase):
+
+class TestOrderBase(unittest.TestCase):
sds: SystemDSContext = None
@@ -45,26 +46,35 @@ class TestOrder(unittest.TestCase):
def tearDownClass(cls):
cls.sds.close()
+
+class TestOrderValid(TestOrderBase):
+
def test_basic(self):
- o = self.sds.from_numpy(m).order(by=by, decreasing=False,
index_return=False).compute()
+ o = self.sds.from_numpy(m).order(
+ by=by, decreasing=False, index_return=False).compute()
s = m[np.argsort(m[:, by-1])]
self.assertTrue(np.allclose(o, s))
def test_index(self):
- o = self.sds.from_numpy(m).order(by=by, decreasing=False,
index_return=True).compute()
+ o = self.sds.from_numpy(m).order(
+ by=by, decreasing=False, index_return=True).compute()
s = np.argsort(m[:, by - 1]) + 1
self.assertTrue(np.allclose(np.transpose(o), s))
def test_decreasing(self):
- o = self.sds.from_numpy(m).order(by=by, decreasing=True,
index_return=True).compute()
+ o = self.sds.from_numpy(m).order(
+ by=by, decreasing=True, index_return=True).compute()
s = np.argsort(-m[:, by - 1]) + 1
self.assertTrue(np.allclose(np.transpose(o), s))
-class TestOrder_1(TestOrder):
+
+class TestOrderInvalid(TestOrderBase):
+
def test_out_of_bounds(self):
by_max = np.size(m, 1) + 2
- with self.assertRaises(RuntimeError) as context:
+ with self.assertRaises(Exception):
self.sds.from_numpy(m).order(by=by_max).compute()
+
if __name__ == "__main__":
unittest.main(exit=False)
diff --git a/src/main/python/tests/matrix/test_print.py
b/src/main/python/tests/matrix/test_print.py
index b7231e7..c7337de 100644
--- a/src/main/python/tests/matrix/test_print.py
+++ b/src/main/python/tests/matrix/test_print.py
@@ -22,6 +22,7 @@
import unittest
import numpy as np
+from time import sleep
from systemds.context import SystemDSContext
@@ -32,6 +33,10 @@ class TestPrint(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.sds = SystemDSContext()
+ sleep(1.0)
+ # Clear stdout ...
+ cls.sds.get_stdout()
+ cls.sds.get_stdout()
@classmethod
def tearDownClass(cls):
diff --git a/src/main/python/tests/script/test_dml_script.py
b/src/main/python/tests/script/test_dml_script.py
index f674057..3c307e1 100644
--- a/src/main/python/tests/script/test_dml_script.py
+++ b/src/main/python/tests/script/test_dml_script.py
@@ -20,7 +20,7 @@
# -------------------------------------------------------------
import unittest
-import time
+from time import sleep
from systemds.context import SystemDSContext
from systemds.script_building import DMLScript
@@ -35,6 +35,9 @@ class Test_DMLScript(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.sds = SystemDSContext()
+ sleep(1)
+ cls.sds.get_stdout()
+ cls.sds.get_stdout()
@classmethod
def tearDownClass(cls):
@@ -44,7 +47,7 @@ class Test_DMLScript(unittest.TestCase):
script = DMLScript(self.sds)
script.add_code('print("Hello")')
script.execute()
- time.sleep(0.5)
+ sleep(0.5)
stdout = self.sds.get_stdout(100)
self.assertListEqual(["Hello"], stdout)
@@ -54,7 +57,7 @@ class Test_DMLScript(unittest.TestCase):
script.add_code('print("World")')
script.add_code('print("!")')
script.execute()
- time.sleep(0.5)
+ sleep(0.5)
stdout = self.sds.get_stdout(100)
self.assertListEqual(['Hello', 'World', '!'], stdout)
@@ -65,7 +68,7 @@ class Test_DMLScript(unittest.TestCase):
scr_a.add_code('y = x + 1')
scr_a.add_code('print(y)')
scr_a.execute()
- time.sleep(0.5)
+ sleep(0.5)
stdout = self.sds.get_stdout(100)
self.assertEqual("4", stdout[0])
self.assertEqual("5", stdout[1])
