This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit ceb4f056cf2ecbf53ff4419a1242e9d6083d6a02
Author: cardosolucas <cardosolucas61....@gmail.com>
AuthorDate: Thu Aug 6 16:46:50 2020 -0300
Update tests to fix CI
---
.../marvin_python_daemon/management/notebook.py | 4 ++--
python-daemon/tests/common/test_data_source_provider.py | 7 ++++---
.../engine_base/serializers/test_keras_serializer.py | 17 +++++++++++------
python-daemon/tests/management/test_notebook.py | 12 ++++++------
4 files changed, 23 insertions(+), 17 deletions(-)
diff --git a/python-daemon/marvin_python_daemon/management/notebook.py
b/python-daemon/marvin_python_daemon/management/notebook.py
index e938575..bf53228 100644
--- a/python-daemon/marvin_python_daemon/management/notebook.py
+++ b/python-daemon/marvin_python_daemon/management/notebook.py
@@ -40,7 +40,7 @@ def notebook(config, enable_security, port):
'extras', 'notebook_extensions',
'jupyter_notebook_config.py')
]
- command.append("--NotebookApp.token=") if not enable_security else None
+ command.append("--NotebookApp.token=") if enable_security else None
command.append("--allow-root")
return_code = os.system(' '.join(command))
@@ -59,7 +59,7 @@ def lab(config, enable_security, port):
'--no-browser'
]
- command.append("--NotebookApp.token=") if not enable_security else None
+ command.append("--NotebookApp.token=") if enable_security else None
command.append("--allow-root")
return_code = os.system(' '.join(command))
diff --git a/python-daemon/tests/common/test_data_source_provider.py
b/python-daemon/tests/common/test_data_source_provider.py
index b54c49d..08d99b2 100644
--- a/python-daemon/tests/common/test_data_source_provider.py
+++ b/python-daemon/tests/common/test_data_source_provider.py
@@ -16,9 +16,9 @@
# limitations under the License.
from marvin_python_daemon.common.data_source_provider import get_spark_session
-from pyspark.tests import ReusedPySparkTestCase
import os
import findspark
+import unittest
findspark.init()
@@ -66,9 +66,10 @@ class TestDataSourceProvider:
)
-class TestSparkDataSource(ReusedPySparkTestCase):
+class TestSparkDataSource(unittest.TestCase):
def test_spark_initialization(self):
- rdd = self.sc.parallelize(['Hi there', 'Hi'])
+ sc = get_spark_session()
+ rdd = sc.sparkContext.parallelize(['Hi there', 'Hi'])
counted = rdd.flatMap(lambda word: word.split(' ')).map(
lambda word: (word, 1)).reduceByKey(lambda acc, n: acc + n)
assert counted.collectAsMap() == {'Hi': 2, 'there': 1}
diff --git
a/python-daemon/tests/engine_base/serializers/test_keras_serializer.py
b/python-daemon/tests/engine_base/serializers/test_keras_serializer.py
index dceb553..79a86c2 100644
--- a/python-daemon/tests/engine_base/serializers/test_keras_serializer.py
+++ b/python-daemon/tests/engine_base/serializers/test_keras_serializer.py
@@ -15,9 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import mock
import pytest
+from tensorflow import keras
from marvin_python_daemon.engine_base import EngineBaseTraining
from marvin_python_daemon.engine_base.serializers.keras_serializer import
KerasSerializer
@@ -31,13 +33,16 @@ def engine():
class TestKerasSerializer(object):
- @mock.patch('tensorflow.keras.models.load_model')
- def test__serializer_load_keras(self, mocked_load, engine):
- mocked_load.return_value = {"me": "here"}
- mocked_path = "/tmp/engine/model"
+ def test__serializer_load_keras(self, engine):
+ mocked_path = os.path.join(os.environ['MARVIN_DATA_PATH'], 'model')
+ a = keras.layers.Input(shape=(2,))
+ x = keras.layers.Dense(3)(a)
+ b = keras.layers.Dense(1)(x)
+ model = keras.models.Model(a, b)
+ model.save(mocked_path)
+
obj = engine._serializer_load(object_file_path=mocked_path)
- mocked_load.assert_called_once_with(mocked_path)
- assert obj == {"me": "here"}
+ assert isinstance(obj, keras.models.Model)
@mock.patch('joblib.load')
def test__serializer_load_not_keras(self, mocked_load, engine):
diff --git a/python-daemon/tests/management/test_notebook.py
b/python-daemon/tests/management/test_notebook.py
index 0172578..2cb7e9b 100644
--- a/python-daemon/tests/management/test_notebook.py
+++ b/python-daemon/tests/management/test_notebook.py
@@ -44,8 +44,8 @@ def test_notebook(system_mocked, sys_mocked):
notebook(config, enable_security, port)
- system_mocked.assert_called_once_with('SPARK_CONF_DIR=/opt/spark/conf
YARN_CONF_DIR=/opt/spark/conf jupyter notebook --notebook-dir /tmp/notebooks
--ip 0.0.0.0 --port 8888 --no-browser --config ' +
- os.environ["MARVIN_ENGINE_PATH"] +
'/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
--NotebookApp.token= --allow-root')
+ system_mocked.assert_called_once_with("SPARK_CONF_DIR={0}
YARN_CONF_DIR={0} jupyter notebook --notebook-dir /tmp/notebooks --ip 0.0.0.0
--port 8888 --no-browser --config
".format(os.path.join(os.environ["SPARK_HOME"], "conf")) +
+ os.environ["MARVIN_ENGINE_PATH"] +
'/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
--allow-root')
@mock.patch('marvin_python_daemon.management.notebook.sys')
@@ -58,8 +58,8 @@ def test_notebook_with_security(system_mocked, sys_mocked):
notebook(config, enable_security, port)
- system_mocked.assert_called_once_with('SPARK_CONF_DIR=/opt/spark/conf
YARN_CONF_DIR=/opt/spark/conf jupyter notebook --notebook-dir /tmp/notebooks
--ip 0.0.0.0 --port 8888 --no-browser --config ' +
- os.environ["MARVIN_ENGINE_PATH"] +
'/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
--allow-root')
+ system_mocked.assert_called_once_with("SPARK_CONF_DIR={0}
YARN_CONF_DIR={0} jupyter notebook --notebook-dir /tmp/notebooks --ip 0.0.0.0
--port 8888 --no-browser --config
".format(os.path.join(os.environ["SPARK_HOME"], "conf")) +
+ os.environ["MARVIN_ENGINE_PATH"] +
'/marvin_python_daemon/extras/notebook_extensions/jupyter_notebook_config.py
--NotebookApp.token= --allow-root')
@mock.patch('marvin_python_daemon.management.notebook.sys')
@@ -74,7 +74,7 @@ def test_jupyter_lab(system_mocked, sys_mocked):
lab(config, enable_security, port)
system_mocked.assert_called_once_with(
- 'SPARK_CONF_DIR=/opt/spark/conf YARN_CONF_DIR=/opt/spark/conf
jupyter-lab --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser
--NotebookApp.token=')
+ "SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter-lab --notebook-dir
/tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser
--allow-root".format(os.path.join(os.environ["SPARK_HOME"], "conf")))
@mock.patch('marvin_python_daemon.management.notebook.sys')
@@ -88,4 +88,4 @@ def test_jupyter_lab_with_security(system_mocked, sys_mocked):
lab(config, enable_security, port)
system_mocked.assert_called_once_with(
- 'SPARK_CONF_DIR=/opt/spark/conf YARN_CONF_DIR=/opt/spark/conf
jupyter-lab --notebook-dir /tmp/notebooks --ip 0.0.0.0 --port 8888
--no-browser')
+ "SPARK_CONF_DIR={0} YARN_CONF_DIR={0} jupyter-lab --notebook-dir
/tmp/notebooks --ip 0.0.0.0 --port 8888 --no-browser --NotebookApp.token=
--allow-root".format(os.path.join(os.environ["SPARK_HOME"], "conf")))
