This is an automated email from the ASF dual-hosted git repository. njayaram pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 8bd5fea5af1402d9de46fd67d5aea63f2a094229 Author: Nandish Jayaram <[email protected]> AuthorDate: Wed May 8 15:17:08 2019 -0700 DL: Add dev-check and unit tests for metrics_compute_frequency JIRA: MADLIB-1335 Add asserts in dev-check and new tests cases in unit-tests for associated functions. Closes #388 Co-authored-by: Ekta Khanna <[email protected]> --- .../modules/deep_learning/test/madlib_keras.sql_in | 37 +++++++++++++++-- .../test/unit_tests/test_madlib_keras.py_in | 46 ++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in index 8567637..5ed3014 100644 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in +++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in @@ -135,6 +135,7 @@ SELECT assert( compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND num_iterations = 3 AND + metrics_compute_frequency = 3 AND num_classes = 2 AND class_values = '{0,1}' AND training_metrics_final >= 0 AND @@ -151,6 +152,35 @@ FROM (SELECT * FROM keras_saved_out_summary) summary; SELECT assert(model_data IS NOT NULL , 'Keras model output validation failed') FROM (SELECT * FROM keras_saved_out) k; + +-- Verify number of iterations for which metrics and loss are computed +DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; +SELECT madlib_keras_fit( + 'cifar_10_sample_batched', + 'keras_saved_out', + 'model_arch', + 1, + $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, + $$ batch_size=2, epochs=1, verbose=0 $$::text, + 7, + NULL, + 'cifar_10_sample_val', + 4); +SELECT assert( + num_iterations = 7 AND + metrics_compute_frequency = 4 AND + training_metrics_final >= 0 AND + training_loss_final >= 0 AND + -- TODO: Uncomment this after MADLIB-1332 is merged to master + -- array_upper(training_metrics, 1) = 2 AND + -- array_upper(training_loss, 1) = 2 AND + -- array_upper(time_iter, 1) = 2 AND + validation_metrics_final >= 0 AND + validation_loss_final >= 0 AND + array_upper(validation_metrics, 1) = 2 AND + array_upper(validation_loss, 1) = 2 , + 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) +FROM (SELECT * FROM keras_saved_out_summary) summary; -- Fit with gpus_per_host set to 2 must error out on machines -- that don't have GPUs. Since Jenkins builds are run on docker containers -- that don't have GPUs, these queries must error out. @@ -202,8 +232,7 @@ SELECT madlib_keras_fit( NULL, NULL, 'model name', 'model desc'); -\x on; -select * from keras_out_summary; + SELECT assert( source_table = 'cifar_10_sample_batched' AND model = 'keras_out' AND @@ -234,8 +263,8 @@ SELECT assert( array_upper(training_loss, 1) = 2 AND validation_metrics_final is NULL AND validation_loss_final is NULL AND - array_upper(validation_metrics, 1) = 0 AND - array_upper(validation_loss, 1) = 0 , + validation_metrics is NULL AND + validation_loss is NULL, 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) FROM (SELECT * FROM keras_out_summary) summary; diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in index 61f7f0a..26fde8c 100644 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in +++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in @@ -386,6 +386,24 @@ class MadlibKerasFitTestCase(unittest.TestCase): def test_fit_final_none(self): result = self.subject.fit_final(None) self.assertEqual(result, None) + def test_should_compute_metrics_this_iter_last_iter_case1(self): + res = self.subject.should_compute_metrics_this_iter(5, 5, 5) + self.assertEqual(True, res) + def test_should_compute_metrics_this_iter_last_iter_case2(self): + res = self.subject.should_compute_metrics_this_iter(5, 3, 5) + self.assertEqual(True, res) + def test_should_compute_metrics_this_iter_before_interval(self): + res = self.subject.should_compute_metrics_this_iter(1, 3, 5) + self.assertEqual(False, res) + def test_should_compute_metrics_this_iter_after_interval(self): + res = self.subject.should_compute_metrics_this_iter(4, 3, 5) + self.assertEqual(False, res) + def test_should_compute_metrics_this_iter_at_interval(self): + res = self.subject.should_compute_metrics_this_iter(3, 3, 5) + self.assertEqual(True, res) + def test_should_compute_metrics_this_iter_every_iter(self): + res = self.subject.should_compute_metrics_this_iter(2, 1, 5) + self.assertEqual(True, res) def test_fit_final_image_count_zero(self): input_state = [0, 0, 0] @@ -733,6 +751,34 @@ class MadlibKerasValidatorTestCase(unittest.TestCase): self.subject._validate_input_shapes( 'dummy_tbl', 'dummy_col', [32,32,3], 1) + def test_is_valid_metrics_compute_frequency_True_None(self): + self.subject.FitInputValidator._validate_input_args = Mock() + obj = self.subject.FitInputValidator( + 'test_table', 'val_table', 'model_table', 'model_arch_table', + 'dep_varname', 'independent_varname', 5, None) + self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) + + def test_is_valid_metrics_compute_frequency_True_num(self): + self.subject.FitInputValidator._validate_input_args = Mock() + obj = self.subject.FitInputValidator( + 'test_table', 'val_table', 'model_table', 'model_arch_table', + 'dep_varname', 'independent_varname', 5, 3) + self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) + + def test_is_valid_metrics_compute_frequency_False_zero(self): + self.subject.FitInputValidator._validate_input_args = Mock() + obj = self.subject.FitInputValidator( + 'test_table', 'val_table', 'model_table', 'model_arch_table', + 'dep_varname', 'independent_varname', 5, 0) + self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) + + def test_is_valid_metrics_compute_frequency_False_greater(self): + self.subject.FitInputValidator._validate_input_args = Mock() + obj = self.subject.FitInputValidator( + 'test_table', 'val_table', 'model_table', 'model_arch_table', + 'dep_varname', 'independent_varname', 5, 6) + self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) + class MadlibSerializerTestCase(unittest.TestCase): def setUp(self): self.plpy_mock = Mock(spec='error')
