This is an automated email from the ASF dual-hosted git repository. nkak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 63847d721c8c12c6b255414a2dc894392cde5617 Author: Nikhil Kak <[email protected]> AuthorDate: Wed Apr 10 16:25:14 2019 -0700 DL: Remove evaluate function JIRA: MADLIB-1304 The `evaluate` UDF keeps all the validation table data in master's memory and then calls keras.eval on it. We can instead run keras.eval on all the segment hosts so that we don't have to keep all data in in one node's memory. We already have a UDF named `evaluate` that is implemented with this logic which will be renamed to `evaluate` in a future PR. This commit removes the evaluate function from the sql as well as the python file. Closes #367 Co-authored-by: Jingyi Mei <[email protected]> --- .../modules/deep_learning/madlib_keras.py_in | 52 ---------------------- .../modules/deep_learning/madlib_keras.sql_in | 27 ----------- 2 files changed, 79 deletions(-) diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in index 83ca5a4..ea77d2e 100644 --- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in +++ b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in @@ -439,58 +439,6 @@ def fit_merge(state1, state2, **kwargs): def fit_final(state, **kwargs): return state -def evaluate(schema_madlib, model_table, source_table, id_col, - model_arch_table, model_arch_id, dependent_varname, - independent_varname, compile_params, output_table, - **kwargs): - module_name = 'madlib_keras_evaluate' - input_tbl_valid(source_table, module_name) - input_tbl_valid(model_arch_table, module_name) - output_tbl_valid(output_table, module_name) - - # _validate_input_args(test_table, model_arch_table, output_table) - device_name = '/cpu:0' - os.environ["CUDA_VISIBLE_DEVICES"] = '-1' - - model_data_query = "SELECT model_data from {0}".format(model_table) - model_data = plpy.execute(model_data_query)[0]['model_data'] - - model_arch_query = "SELECT model_arch, model_weights FROM {0} " \ - "WHERE id = {1}".format(model_arch_table, model_arch_id) - query_result = plpy.execute(model_arch_query) - - query_result = query_result[0] - model_arch = query_result['model_arch'] - input_shape = get_input_shape(model_arch) - model = model_from_json(model_arch) - - model_shapes = [] - for weight_arr in model.get_weights(): - model_shapes.append(weight_arr.shape) - _, updated_weights = madlib_keras_serializer.deserialize_weights( - model_data, model_shapes) - model.set_weights(updated_weights) - optimizers = get_optimizers() - (opt_name,final_args,compile_dict) = parse_compile_params(compile_params) - with K.tf.device(device_name): - model.compile(optimizer=optimizers[opt_name](**final_args), - loss=compile_dict['loss'], - metrics=compile_dict['metrics']) - - input_shape = map(int, input_shape) - x_validation, y_validation = get_data_as_np_array(source_table, - dependent_varname, - independent_varname) - - plpy.info('X shape : {0}'.format(x_validation.shape)) - plpy.info('Y shape : {0}'.format(y_validation.shape)) - - with K.tf.device(device_name): - evaluate_result = model.evaluate(x_validation, y_validation) - - plpy.info('evaluate result is {}'.format(evaluate_result)) - - def evaluate1(schema_madlib, model_table, test_table, id_col, model_arch_table, model_arch_id, dependent_varname, independent_varname, compile_params, output_table, **kwargs): diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in index 9526c91..37afad9 100644 --- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in +++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in @@ -205,33 +205,6 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_predict( $$ LANGUAGE plpythonu VOLATILE m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate( - model_table VARCHAR, - test_table VARCHAR, - id_col VARCHAR, - model_arch_table VARCHAR, - model_arch_id INTEGER, - dependent_varname VARCHAR, - independent_varname VARCHAR, - compile_params VARCHAR, - output_table VARCHAR -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras') - with AOControl(False): - madlib_keras.evaluate(schema_madlib, - model_table, - test_table, - id_col, - model_arch_table, - model_arch_id, - dependent_varname, - independent_varname, - compile_params, - output_table) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate1( model_table VARCHAR, test_table VARCHAR,
