This is an automated email from the ASF dual-hosted git repository. okislal pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/madlib.git
commit e6bad2fac706fd7c1e49a5249215195555b9f1e3 Author: Orhan Kislal <[email protected]> AuthorDate: Fri Jan 17 12:26:32 2020 -0500 Build: Update madpack scripts for pg11 and add changelist Co-authored-by: Amil Khanzada <[email protected]> --- src/madpack/changelist_1.16_1.17.yaml | 142 ++++++++++++++++++++++++++++++++++ src/madpack/diff_udf.sql | 120 ++++++++++++++-------------- src/madpack/madpack.py | 3 + src/madpack/upgrade_util.py | 16 +++- 4 files changed, 221 insertions(+), 60 deletions(-) diff --git a/src/madpack/changelist_1.16_1.17.yaml b/src/madpack/changelist_1.16_1.17.yaml new file mode 100644 index 0000000..90f2e88 --- /dev/null +++ b/src/madpack/changelist_1.16_1.17.yaml @@ -0,0 +1,142 @@ +# ------------------------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------------------------ + +# Changelist for MADlib version rel/v1.16 to rel/v1.17.0 + +# This file contains all changes that were introduced in a new version of +# MADlib. This changelist is used by the upgrade script to detect what objects +# should be upgraded (while retaining all other objects from the previous version) + +# New modules (actually .sql_in files) added in upgrade version +# For these files the sql_in code is retained as is with the functions in the +# file installed on the upgrade version. All other files (that don't have +# updates), are cleaned up to remove object replacements +new module: + madlib_keras_fit_multiple_model: + madlib_keras_gpu_info: + madlib_keras_model_selection: + +# Changes in the types (UDT) including removal and modification +udt: + +# List of the UDF changes that affect the user externally. This includes change +# in function name, return type, argument order or types, or removal of +# the function. In each case, the original function is as good as removed and a +# new function is created. In such cases, we should abort the upgrade if there +# are user views dependent on this function, since the original function will +# not be present in the upgraded version. +udf: + - fit_final_multiple_model: + rettype: bytea + argument: bytea + - fit_transition: + rettype: bytea + argument: bytea, smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea + - graph_apsp_get_path: + rettype: void + argument: text, integer, integer, text + - graph_sssp: + rettype: void + argument: text, text, text, text, integer, text + - graph_sssp: + rettype: void + argument: text, text, text, text, integer, text, text + - graph_sssp_get_path: + rettype: void + argument: text, integer, text + - internal_keras_eval_transition: + rettype: real[] + argument: real[], smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer + - internal_keras_predict: + rettype: double precision[] + argument: real[], text, bytea, boolean, double precision, integer, integer[], integer[], integer, integer + - madlib_keras_evaluate: + rettype: void + argument: character varying, character varying, character varying, integer + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying + - madlib_keras_fit: + rettype: void + argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying, character varying + - madlib_keras_predict: + rettype: void + argument: character varying, character varying, character varying, character varying, character varying, character varying, integer + - pivot: + rettype: void + argument: text, text, text, text, text, text, text, boolean, boolean, text + - pivot: + rettype: void + argument: text, text, text, text, text, text, text, boolean, boolean + - pivot: + rettype: void + argument: text, text, text, text, text, text, text, boolean + - pivot: + rettype: void + argument: text, text, text, text, text, text, text + - pivot: + rettype: void + argument: text, text, text, text, text, text, boolean + - pivot: + rettype: void + argument: text, text, text, text, text, text + - pivot: + rettype: void + argument: text, text, text, text, text, boolean + - pivot: + rettype: void + argument: text, text, text, text, text + + +# Changes to aggregates (UDA) including removal and modification +# Overloaded functions should be mentioned separately +uda: + - agg_array_concat: + rettype: anyarray + argument: anyarray + - fit_step: + rettype: bytea + argument: smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea + - internal_keras_evaluate: + rettype: real[] + argument: smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer + +# List of the UDC, UDO and UDOC changes. +udc: + +# Changes in the operators (UDO) +udo: + +# Changes in the operator classes (UDOC) +udoc: diff --git a/src/madpack/diff_udf.sql b/src/madpack/diff_udf.sql index d73093c..3226409 100644 --- a/src/madpack/diff_udf.sql +++ b/src/madpack/diff_udf.sql @@ -15,67 +15,75 @@ CREATE OR REPLACE FUNCTION get_functions(table_name text, schema_name text, type_filter text) RETURNS VOID AS $$ - import plpy - plpy.execute(""" - CREATE TABLE {table_name} AS - SELECT - "schema", "name", filter_schema("retype", '{schema_name}') retype, - filter_schema("argtypes", '{schema_name}') argtypes, "type" - FROM - ( +import plpy +column_name = plpy.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_schema='pg_catalog' and table_name='pg_proc' and + (column_name='prokind' or column_name='proisagg') + """)[0]['column_name'] +proisagg_wrapper = "p.proisagg" if column_name == 'proisagg' else "p.prokind = 'a'" +plpy.execute(""" + CREATE TABLE {table_name} AS + SELECT + "schema", "name", filter_schema("retype", '{schema_name}') retype, + filter_schema("argtypes", '{schema_name}') argtypes, "type" + FROM + ( - SELECT n.nspname as "schema", - p.proname as "name", - CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END || - pg_catalog.format_type(p.prorettype, NULL) as "retype", - CASE WHEN proallargtypes IS NOT NULL THEN - pg_catalog.array_to_string(ARRAY( + SELECT n.nspname as "schema", + p.proname as "name", + CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END || + pg_catalog.format_type(p.prorettype, NULL) as "retype", + CASE WHEN proallargtypes IS NOT NULL THEN + pg_catalog.array_to_string(ARRAY( + SELECT + pio || ptyp + FROM + ( SELECT - pio || ptyp - FROM - ( - SELECT - CASE - WHEN p.proargmodes[s.i] = 'i' THEN '' - WHEN p.proargmodes[s.i] = 'o' THEN 'OUT ' - WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT ' - WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC ' - END AS pio, - --CASE - -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN '' - -- ELSE p.proargnames[s.i] || ' ' - --END || - pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp - FROM - pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i) - ) qx - WHERE pio = '' - ), ', ') - ELSE - pg_catalog.array_to_string(ARRAY( - SELECT + CASE + WHEN p.proargmodes[s.i] = 'i' THEN '' + WHEN p.proargmodes[s.i] = 'o' THEN 'OUT ' + WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT ' + WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC ' + END AS pio, --CASE - -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN '' - -- ELSE p.proargnames[s.i+1] || ' ' - -- END || - pg_catalog.format_type(p.proargtypes[s.i], NULL) + -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN '' + -- ELSE p.proargnames[s.i] || ' ' + --END || + pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp FROM - pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i) + pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i) + ) qx + WHERE pio = '' ), ', ') - END AS "argtypes", - CASE - WHEN p.proisagg THEN 'agg' - WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' - ELSE 'normal' - END AS "type" - FROM pg_catalog.pg_proc p - LEFT JOIN pg_catalog.pg_namespace n - ON n.oid = p.pronamespace - WHERE n.nspname ~ '^({schema_name})$' - ORDER BY 1, 2, 4 - ) q - WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]' - """.format(table_name=table_name, schema_name=schema_name, type_filter=type_filter)) + ELSE + pg_catalog.array_to_string(ARRAY( + SELECT + --CASE + -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN '' + -- ELSE p.proargnames[s.i+1] || ' ' + -- END || + pg_catalog.format_type(p.proargtypes[s.i], NULL) + FROM + pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i) + ), ', ') + END AS "argtypes", + CASE + WHEN {proisagg_wrapper} THEN 'agg' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END AS "type" + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n + ON n.oid = p.pronamespace + WHERE n.nspname ~ '^({schema_name})$' + ORDER BY 1, 2, 4 + ) q + WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]' + """.format(table_name=table_name, schema_name=schema_name, + type_filter=type_filter, proisagg_wrapper=proisagg_wrapper)) $$ LANGUAGE plpythonu; diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py index 86a7236..2af0480 100755 --- a/src/madpack/madpack.py +++ b/src/madpack/madpack.py @@ -701,6 +701,9 @@ def _process_py_sql_files_in_modules(modset, args_dict): args_dict['create_obj_handle'], args_dict['sc']) elif calling_operation == INSTALL_DEV_CHECK: + # Skip certain tests for GP4.3 + if dbver == '4.3ORCA' and module in ['deep_learning', 'kmeans']: + continue _execute_per_module_install_dev_check_algo( args_dict['schema'], args_dict['test_user'], diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py index 90511a3..9197b99 100644 --- a/src/madpack/upgrade_util.py +++ b/src/madpack/upgrade_util.py @@ -502,6 +502,9 @@ class ViewDependency(UpgradeBase): """ @brief Detect direct view dependencies on MADlib UDFs/UDAs """ + proisagg_wrapper = "p.proisagg" + if self._portid == 'postgres' and self._dbver > 11: + proisagg_wrapper = "p.prokind = 'a'" rows = self._run_sql(""" SELECT view, nsp.nspname AS schema, procname, procoid, proisagg @@ -513,7 +516,7 @@ class ViewDependency(UpgradeBase): c.relnamespace AS namespace, p.proname As procname, p.oid AS procoid, - p.proisagg AS proisagg + {proisagg_wrapper} AS proisagg FROM pg_class AS c, pg_rewrite AS rw, @@ -529,7 +532,8 @@ class ViewDependency(UpgradeBase): ) t1 WHERE t1.namespace = nsp.oid - """.format(schema_madlib_oid=self._schema_oid)) + """.format(schema_madlib_oid=self._schema_oid, + proisagg_wrapper=proisagg_wrapper)) self._view2proc = defaultdict(list) for row in rows: @@ -1029,6 +1033,10 @@ class ScriptCleaner(UpgradeBase): """ # See _get_function_info for explanations. + proisagg_wrapper = "p.proisagg = true" + if self._portid == 'postgres' and self._dbver > 11: + proisagg_wrapper = "p.prokind = 'a'" + rows = self._run_sql(""" SELECT max(proname) AS proname, @@ -1047,12 +1055,12 @@ class ScriptCleaner(UpgradeBase): pg_namespace AS nsp WHERE p.pronamespace = nsp.oid AND - p.proisagg = true AND + {proisagg_wrapper} AND nsp.nspname = '{schema}' ) AS f GROUP BY procoid - """.format(schema=self._schema)) + """.format(schema=self._schema, proisagg_wrapper=proisagg_wrapper)) self._existing_uda = defaultdict(list) for row in rows: # Consider about the overloaded aggregates
