This is an automated email from the ASF dual-hosted git repository. njayaram pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 64ad1d45776db5b1e3420196bd097e9d04ade33a Author: Domino Valdano <[email protected]> AuthorDate: Thu Mar 14 12:22:50 2019 -0700 New Module: Deep Learning helper functions JIRA: MADLIB-1306 This commit introduces two new helper functions, load_keras_model() and delete_keras_model(). It is a part of the new deep learning functionality being added to madlib. These functions will help prepare the input to the keras_fit() function being introduced in PR #355. keras_fit() takes a model_arch_table param and a model_id param. This refers to a row in a "keras model arch table" which has to have a particular format. These functions will help the user to create and manage that table, without worrying about the details of the format. Each row of the keras model arch table represents a keras model architecture, and is identified by a model_id. load_keras_model() will add a new model architecture to the table, creating the table if it doesn't exist and then inserting the appropriate row into it. The newly assigned model_id will be printed for the user, who can then pass that as the model_id parameter to keras_fit(). The model architectures themselves are expected to be in JSON format, which can be achieved by calling model.to_json() on any keras model after it is created (in python). The keras model arch table also has a column to store weights. This is initialized to null by load_keras_model() but we plan to add another helper function in the near future to be able to set the weights (useful for doing a warm start, instead of letting keras randomly initialize the weights when training begins). The keras model arch table itself will be automatically dropped by delete_keras_model() after all of the models in it are deleted and the table is empty. To help avoid mistakes, if either of these functions is run on a table that doesn't match the expected format, and exception will be thrown and the table will be left untouched. Co-authored-by: Ekta Khanna <[email protected]> --- .../modules/convex/keras_model_arch_table.py_in | 133 +++++++++++++++++++++ .../modules/convex/keras_model_arch_table.sql_in | 39 ++++++ .../convex/test/keras_model_arch_table.ic.sql_in | 31 +++++ .../convex/test/keras_model_arch_table.sql_in | 124 +++++++++++++++++++ 4 files changed, 327 insertions(+) diff --git a/src/ports/postgres/modules/convex/keras_model_arch_table.py_in b/src/ports/postgres/modules/convex/keras_model_arch_table.py_in new file mode 100644 index 0000000..f6d8b41 --- /dev/null +++ b/src/ports/postgres/modules/convex/keras_model_arch_table.py_in @@ -0,0 +1,133 @@ +# coding=utf-8 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +@file keras_model_arch_table.py_in + +@brief keras model arch table management helper functions + +@namespace keras_model_arch_table +""" + +from utilities.validate_args import table_exists +from utilities.validate_args import columns_missing_from_table +from utilities.validate_args import quote_ident +from utilities.control import MinWarning +from internal.db_utils import quote_literal +from utilities.utilities import unique_string +import plpy + +class Format: + """Expected format of keras_model_arch_table. + Example uses: + + from utilities.validate_args import columns_missing_from_table + from keras_model_arch_table import Format + + # Validate names in cols list against actual table + missing_cols = columns_missing_from_table('my_arch_table', Format.col_names) + + # Get model arch from keras model arch table, without hard coding column names + sql = "SELECT {arch} FROM {table} WHERE {id} = {my_id}" + .format(arch=Format.model_arch, + table='my_arch_table', + id=Format.model_id, + my_id=1) + arch = plpy.execute(sql)[0] + + """ + col_names = ('model_id', 'model_arch', 'model_weights', '__internal_madlib_id__') + col_types = ('SERIAL PRIMARY KEY', 'JSON', 'DOUBLE PRECISION[]', 'TEXT') + (model_id, model_arch, model_weights, __internal_madlib_id__) = col_names + +@MinWarning("warning") +def _execute(sql,max_rows=0): + return plpy.execute(sql,max_rows) + +def load_keras_model(keras_model_arch_table, model_arch,**kwargs): + model_arch_table = quote_ident(keras_model_arch_table) + if not table_exists(model_arch_table): + col_defs = ','.join(map(' '.join, + zip(Format.col_names, + Format.col_types))) + + sql = "CREATE TABLE {model_arch_table} ({col_defs})" \ + .format(**locals()) + + _execute(sql) + plpy.info("Created new keras model arch table {0}." \ + .format(model_arch_table)) + else: + missing_cols = columns_missing_from_table(model_arch_table, Format.col_names) + if len(missing_cols) > 0: + plpy.error("Invalid keras model arch table {0}," + " missing columns: {1}".format(model_arch_table, missing_cols)) + + unique_str = unique_string(prefix_has_temp=False) + + sql = """INSERT INTO {model_arch_table} ({model_arch_col}, {internal_id_col}) + VALUES({model_arch}, '{unique_str}'); + SELECT model_id, model_arch + FROM {model_arch_table} WHERE {internal_id_col} = '{unique_str}' + """.format(model_arch_table=model_arch_table, + model_arch_col=Format.model_arch, + unique_str=unique_str, + model_arch=quote_literal(model_arch), + internal_id_col=Format.__internal_madlib_id__) + +# This code works perfectly in postgres 8.3+, but fails in Greenplum 5 with: +# ERROR: The RETURNING clause of the INSERT statement is not supported in this version +# of Greenplum Database. +# sql = """INSERT INTO {model_arch_table} +# (model_arch) +# VALUES('{model_arch}') +# RETURNING *;""".format(model_arch_table=model_arch_table, +# model_arch=quote_literal(model_arch)) + res = _execute(sql,1) + + if len(res) != 1 or res[0][Format.model_arch] != model_arch: + raise Exception("Failed to insert new row in {0} table--try again?" + .format(model_arch_table)) + plpy.info("Added model id {0} to {1} table".format(res[0]['model_id'], model_arch_table)) + +def delete_keras_model(keras_model_arch_table, model_id, **kwargs): + model_arch_table = quote_ident(keras_model_arch_table) + if not table_exists(model_arch_table): + plpy.error("Table {0} does not exist.".format(model_arch_table)) + + missing_cols = columns_missing_from_table(model_arch_table, Format.col_names) + if len(missing_cols) > 0: + plpy.error("Invalid keras model arch table {0}," + " missing columns: {1}".format(model_arch_table, missing_cols)) + + sql = """ + DELETE FROM {model_arch_table} WHERE model_id={model_id} + """.format(model_arch_table=model_arch_table, model_id=model_id) + res = _execute(sql) + + if res.nrows() > 0: + plpy.info("Model id {0} has been deleted from {1}.".format(model_id, model_arch_table)) + else: + plpy.info("Model id {0} not found".format(model_id)) + + sql = "SELECT model_id FROM {model_arch_table}".format(model_arch_table=model_arch_table) + res = _execute(sql) + if not res or len(res) == 0: + plpy.info("Removing empty keras model arch table {model_arch_table}".format(model_arch_table=model_arch_table)) + sql = "DROP TABLE {model_arch_table}".format(model_arch_table=model_arch_table) + _execute(sql) diff --git a/src/ports/postgres/modules/convex/keras_model_arch_table.sql_in b/src/ports/postgres/modules/convex/keras_model_arch_table.sql_in new file mode 100644 index 0000000..b27b979 --- /dev/null +++ b/src/ports/postgres/modules/convex/keras_model_arch_table.sql_in @@ -0,0 +1,39 @@ +/* ----------------------------------------------------------------------- *//** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * + * @file model_arch_table.sql_in + * + * @brief SQL functions for multilayer perceptron + * @date June 2012 + * + * + *//* ----------------------------------------------------------------------- */ + +m4_include(`SQLCommon.m4') + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(keras_model_arch_table VARCHAR, model_arch JSON) +RETURNS VOID AS $$ + PythonFunction(`convex',`keras_model_arch_table',`load_keras_model') + $$ LANGUAGE plpythonu VOLATILE; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model(keras_model_arch_table VARCHAR, model_id INTEGER) +RETURNS VOID AS $$ + PythonFunction(`convex',`keras_model_arch_table',`delete_keras_model') +$$ LANGUAGE plpythonu VOLATILE; diff --git a/src/ports/postgres/modules/convex/test/keras_model_arch_table.ic.sql_in b/src/ports/postgres/modules/convex/test/keras_model_arch_table.ic.sql_in new file mode 100644 index 0000000..8d82a43 --- /dev/null +++ b/src/ports/postgres/modules/convex/test/keras_model_arch_table.ic.sql_in @@ -0,0 +1,31 @@ +/* ----------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ----------------------------------------------------------------------- */ + +/* ----------------------------------------------------------------------------- + * Test Keras Model Arch Table helper functions + * -------------------------------------------------------------------------- */ + +SELECT load_keras_model('test239587_keras_model_arch_table','{"a" : 1, "b" : 2, "c" : [4,5,6] }'); +SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [1,2,3]}'); +SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [8,4,0]}'); +SELECT delete_keras_model('test239587_keras_model_arch_table',2); +SELECT delete_keras_model('test239587_keras_model_arch_table',3); +SELECT delete_keras_model('test239587_keras_model_arch_table',1); diff --git a/src/ports/postgres/modules/convex/test/keras_model_arch_table.sql_in b/src/ports/postgres/modules/convex/test/keras_model_arch_table.sql_in new file mode 100644 index 0000000..64b34b0 --- /dev/null +++ b/src/ports/postgres/modules/convex/test/keras_model_arch_table.sql_in @@ -0,0 +1,124 @@ +/* ----------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ----------------------------------------------------------------------- */ + +/* ----------------------------------------------------------------------------- + * Test Keras Model Arch Table helper functions + * -------------------------------------------------------------------------- */ + + +/* Test successful model creation where no table exists */ +DROP TABLE IF EXISTS test_keras_model_arch_table; +SELECT load_keras_model('test_keras_model_arch_table', '{"a" : 1, "b" : 2, "c" : [4,5,6] }'); + +SELECT assert(UPPER(atttypid::regtype::TEXT) = 'INTEGER', 'model_id column should be INTEGER type') + FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass + AND attname = 'model_id'; +SELECT assert(UPPER(atttypid::regtype::TEXT) = 'JSON', 'model_arch column should be JSON type' ) FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass + AND attname = 'model_arch'; +SELECT assert(UPPER(atttypid::regtype::TEXT) = + 'DOUBLE PRECISION[]', 'model_weights column should be DOUBLE PRECISION[] type') + FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass + AND attname = 'model_weights'; + +/* model id should be 1 */ +SELECT assert(model_id = 1, 'Wrong model_id written by load_keras_model') + FROM test_keras_model_arch_table; + +/* model arch should be valid json, with all fields accessible with json operators */ +SELECT assert((model_arch->>'a') = '1', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table; +SELECT assert((model_arch->>'b') = '2', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table; +SELECT assert((model_arch->'c')->>0 = '4', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table; +SELECT assert((model_arch->'c')->>1 = '5', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table; +SELECT assert((model_arch->'c')->>2 = '6', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table; +/* model_weights should be set to null, since this is not a warm start */ +SELECT assert(model_weights IS NULL, 'model_weights should be NULL after load_keras_model() called.') FROM test_keras_model_arch_table; + + +/* Test model creation where valid table exists */ +SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); +SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [8,4,0]}'); +SELECT assert(model_arch->'config'->>0 = '1', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 2; +SELECT assert(model_arch->'config'->>1 = '2', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 2; +SELECT assert(model_arch->'config'->>2 = '3', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 2; +SELECT assert(model_arch->'config'->>0 = '8', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 3; +SELECT assert(model_arch->'config'->>1 = '4', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 3; +SELECT assert(model_arch->'config'->>2 = '0', 'Cannot parse model_arch json in model table.') + FROM test_keras_model_arch_table WHERE model_id = 3; + +/* Test deletion where valid table exists */ +SELECT delete_keras_model('test_keras_model_arch_table', 2); +SELECT assert(COUNT(model_id) = 0, 'model id 2 should have been deleted!') + FROM test_keras_model_arch_table WHERE model_id = 2; +SELECT delete_keras_model('test_keras_model_arch_table', 3); +SELECT assert(COUNT(model_id) = 0, 'model id 3 should have been deleted!') + FROM test_keras_model_arch_table WHERE model_id = 3; + /* Delete a second time, to make sure nothing weird happens. + * It should archrt to the user that the model_id wasn't found but not + * raise an exception or change anything. */ +SELECT delete_keras_model('test_keras_model_arch_table', 3); +SELECT assert(COUNT(model_id) = 0, 'model id 3 should have been deleted!') + FROM test_keras_model_arch_table WHERE model_id = 3; +SELECT delete_keras_model('test_keras_model_arch_table', 1); +SELECT assert(COUNT(relname) = 0, 'Table test_keras_model_arch_table should have been deleted.') + FROM pg_class where relname = 'test_keras_model_arch_table'; + +SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); +DELETE FROM test_keras_model_arch_table; + +/* Test deletion where empty table exists */ +SELECT delete_keras_model('test_keras_model_arch_table', 3); +SELECT assert(COUNT(relname) = 0, 'Table test_keras_model_arch_table should have been deleted.') from pg_class where relname = 'test_keras_model_arch_table'; + +/* Test deletion where invalid table exists */ + +SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); +ALTER TABLE test_keras_model_arch_table DROP COLUMN model_id; +CREATE FUNCTION trap_error(stmt TEXT) RETURNS INTEGER AS $$ +BEGIN + BEGIN + EXECUTE stmt; + EXCEPTION + WHEN OTHERS THEN + RETURN 1; + END; + RETURN 0; +END; +$$ LANGUAGE plpgsql; + +SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 1)$$) = 1, + 'Deleting an invalid table should generate an exception.'); + +SELECT assert(trap_error($$SELECT load_keras_model('test_keras_model_arch_table', '{"config" : 1}')$$) = 1, 'Passing an invalid table to load_keras_model() should raise exception.'); + +/* Test deletion where no table exists */ +DROP TABLE test_keras_model_arch_table; +SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 3)$$) = 1, + 'Deleting a non-existent table should raise exception.');
