orhankislal commented on a change in pull request #498: URL: https://github.com/apache/madlib/pull/498#discussion_r426916245
########## File path: src/ports/postgres/modules/deep_learning/madlib_keras_custom_function.py_in ########## @@ -0,0 +1,131 @@ +# coding=utf-8 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations + +import dill +import plpy +from plpy import spiexceptions +from utilities.control import MinWarning +from utilities.utilities import _assert +from utilities.utilities import get_col_name_type_sql_string +from utilities.utilities import unique_string +from utilities.validate_args import columns_missing_from_table +from utilities.validate_args import input_tbl_valid +from utilities.validate_args import quote_ident +from utilities.validate_args import table_exists + +module_name = 'Keras Custom Funtion' +class CustomFunctionSchema: + """Expected format of custom function table. + Example uses: + + from utilities.validate_args import columns_missing_from_table + from madlib_keras_custom_function import CustomFunctionSchema + + # Validate names in cols list against actual table + missing_cols = columns_missing_from_table('my_custom_fn_table', CustomFunctionSchema.col_names) + + # Get function object from table, without hard coding column names + sql = "SELECT {object} FROM {table} WHERE {id} = {my_id}" + .format(object=CustomFunctionSchema.FN_OBJ, + table='my_custom_fn_table', + id=CustomFunctionSchema.FN_ID, + my_id=1) + object = plpy.execute(sql)[0] + + """ + FN_ID = 'id' + FN_NAME = 'name' + FN_OBJ = 'object' + FN_DESC = 'description' + col_names = (FN_ID, FN_NAME, FN_DESC, FN_OBJ) + col_types = ('SERIAL', 'TEXT', 'TEXT', 'BYTEA') + +def _validate_object(object, **kwargs): + _assert(object is not None, "{0}: function object cannot be NULL!".format(module_name)) + try: + obj=dill.loads(object) + except Exception as e: + plpy.error("{0}: Invalid function object".format(module_name, e)) + +@MinWarning("error") +def load_custom_function(object_table, object, name, description=None, **kwargs): + object_table = quote_ident(object_table) + _validate_object(object) + _assert(name is not None, + "{0}: function name cannot be NULL!".format(module_name)) + if not table_exists(object_table): + col_defs = get_col_name_type_sql_string(CustomFunctionSchema.col_names, + CustomFunctionSchema.col_types) + + sql = "CREATE TABLE {0} ({1}, PRIMARY KEY({2}))" \ + .format(object_table, col_defs, CustomFunctionSchema.FN_NAME) + + plpy.execute(sql, 0) + plpy.info("{0}: Created new custom function table {1}." \ + .format(module_name, object_table)) + else: + missing_cols = columns_missing_from_table(object_table, + CustomFunctionSchema.col_names) + if len(missing_cols) > 0: + plpy.error("{0}: Invalid custom function table {1}," + " missing columns: {2}".format(module_name, + object_table, + missing_cols)) + + insert_query = plpy.prepare("INSERT INTO {0} " + "VALUES(DEFAULT, $1, $2, $3);".format(object_table), + CustomFunctionSchema.col_types[1:]) + try: + plpy.execute(insert_query,[name, description, object], 0) + except spiexceptions.UniqueViolation: + plpy.error("Function '{0}' already exists in {1}".format(name, object_table)) + except Exception as e: + plpy.error(e) + + plpy.info("{0}: Added function {1} to {2} table". + format(module_name, name, object_table)) + +@MinWarning("error") +def delete_custom_function(object_table, id, **kwargs): + object_table = quote_ident(object_table) + input_tbl_valid(object_table, "Keras Custom Funtion") + + missing_cols = columns_missing_from_table(object_table, CustomFunctionSchema.col_names) + if len(missing_cols) > 0: + plpy.error("{0}: Invalid custom function table {1}," + " missing columns: {2}".format(module_name, object_table, + missing_cols)) + + sql = """ + DELETE FROM {0} WHERE {1}={2} + """.format(object_table, CustomFunctionSchema.FN_ID, id) + res = plpy.execute(sql, 0) + + if res.nrows() > 0: + plpy.info("{0}: Object id {1} has been deleted from {2}.". + format(module_name, id, object_table)) + else: + plpy.error("{0}: Object id {1} not found".format(module_name, id)) + + sql = "SELECT {0} FROM {1}".format(CustomFunctionSchema.FN_ID, object_table) + res = plpy.execute(sql, 0) + if not res: + plpy.info("{0}: Dropping empty keras function table " \ Review comment: `Dropping empty keras function table` -> `Dropping empty **custom** keras function table` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
