This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
The following commit(s) were added to refs/heads/master by this push:
new 5014eec Build: Remove function dependencies from changelist
5014eec is described below
commit 5014eec5a3375647f86fc9b9aa67f4424cdb3a0b
Author: Orhan Kislal <[email protected]>
AuthorDate: Tue Apr 21 19:30:47 2020 -0400
Build: Remove function dependencies from changelist
Currently the madpack checks if there are any dependencies on madlib
functions
(and aggregates). This has 2 steps
1. Check if there is a dependency on a madlib function
2. Check if the function is included in the changelist
If both conditions are satisfied, madpack stops the upgrade and gives a
proper
message to the user including the dependency chain.
This commit removes the second step (changelist based check) and stop the
upgrade
if there are any dependencies on madlib functions. It will ensure that our
upgrades
are less error prone and the user will still have the dependency chain.
---
src/madpack/changelist_1.17.0_1.18.0-dev.yaml | 54 +++++++++++++
src/madpack/madpack.py | 105 ++++++++++++++++++++------
src/madpack/upgrade_util.py | 16 +---
src/madpack/utilities.py | 1 -
4 files changed, 141 insertions(+), 35 deletions(-)
diff --git a/src/madpack/changelist_1.17.0_1.18.0-dev.yaml
b/src/madpack/changelist_1.17.0_1.18.0-dev.yaml
new file mode 100644
index 0000000..3df7f4c
--- /dev/null
+++ b/src/madpack/changelist_1.17.0_1.18.0-dev.yaml
@@ -0,0 +1,54 @@
+#
------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
------------------------------------------------------------------------------
+
+# Changelist for MADlib version rel/v1.17.0 to rel/v1.18.0-dev
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous
version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# List of the UDC, UDO and UDOC changes.
+udc:
+
+# Changes in the operators (UDO)
+udo:
+
+# Changes in the operator classes (UDOC)
+udoc:
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 2af0480..f620a68 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -518,40 +518,34 @@ def _db_upgrade(schema, filehandle, db_madlib_ver):
info_(this, vd.get_dependency_graph_str(), True)
info_(this, "*" * 50, True)
- c_udf = ch.get_udf_signature()
d_udf = vd.get_depended_func_signature('UDF')
- cd_udf = [udf for udf in d_udf if udf in c_udf]
- if len(cd_udf) > 0:
+ if len(d_udf) > 0:
error_(this, """
User has objects dependent on following updated MADlib
functions!
{0}
- These objects will fail to work with the updated functions and
+ These objects might fail to work with the updated functions and
need to be dropped before starting upgrade again.
- """.format('\n\t\t\t\t\t'.join(cd_udf)), False)
+ """.format('\n\t\t\t\t\t'.join(d_udf)), False)
abort = True
- c_uda = ch.get_uda_signature()
d_uda = vd.get_depended_func_signature('UDA')
- cd_uda = [uda for uda in d_uda if uda in c_uda]
- if len(cd_uda) > 0:
+ if len(d_uda) > 0:
error_(this, """
User has objects dependent on following updated MADlib
aggregates!
{0}
- These objects will fail to work with the new aggregates and
+ These objects might fail to work with the new aggregates and
need to be dropped before starting upgrade again.
- """.format('\n\t\t\t\t\t'.join(cd_uda)), False)
+ """.format('\n\t\t\t\t\t'.join(d_uda)), False)
abort = True
- c_udo = ch.get_udo_oids()
d_udo = vd.get_depended_opr_oids()
- cd_udo = [udo for udo in d_udo if udo in c_udo]
- if len(cd_udo) > 0:
+ if len(d_udo) > 0:
error_(this, """
User has objects dependent on following updated MADlib
operators!
oid={0}
- These objects will fail to work with the new operators and
+ These objects might fail to work with the new operators and
need to be dropped before starting upgrade again.
- """.format('\n\t\t\t\t\t'.join(cd_udo)), False)
+ """.format('\n\t\t\t\t\t'.join(d_udo)), False)
abort = True
if abort:
@@ -565,13 +559,15 @@ def _db_upgrade(schema, filehandle, db_madlib_ver):
sc = uu.ScriptCleaner(schema, portid, con_args, ch)
info_(this, "Script Cleaner initialized ...", False)
- ch.drop_changed_uda()
- ch.drop_changed_udoc()
- ch.drop_changed_udo()
+ function_drop_str = get_madlib_function_drop_str(schema)
+ flist = function_drop_str.split("\n\n")
+ for i in flist:
+ _internal_run_query(i, True)
+
+ operator_drop_str = get_madlib_operator_drop_str(schema)
+ _internal_run_query(operator_drop_str, True)
ch.drop_changed_udc()
- ch.drop_changed_udf()
ch.drop_changed_udt() # assume dependent udf for udt does not change
- ch.drop_traininginfo_4dt() # used types: oid, text, integer, float
_db_create_objects(schema, filehandle, True, sc)
return 0
#
------------------------------------------------------------------------------
@@ -1200,10 +1196,77 @@ def create_install_madlib_sqlfile(args, madpack_cmd):
else:
# 3) Run upgrade
_plpy_check(py_min_ver)
+
return_signal = _db_upgrade(schema, output_filehandle,
db_madlib_ver)
return 1 if return_signal > 0 else 0
+def get_madlib_function_drop_str(schema):
+
+ if portid == 'greenplum':
+ case_str = """
+ CASE
+ WHEN p.proisagg THEN 'aggregate'
+ ELSE 'function'
+ """
+ else:
+ case_str = """
+ CASE p.prokind
+ WHEN 'a' THEN 'aggregate'
+ ELSE 'function'
+ """
+ madlib_functions = _internal_run_query("""
+ SELECT n.nspname as "Schema",
+ p.proname as "name",
+ pg_catalog.pg_get_function_arguments(p.oid) as "args",
+ {0}
+ END as "type"
+ FROM pg_catalog.pg_proc p
+ LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+ WHERE n.nspname OPERATOR(pg_catalog.~) '^({1})$'
+ ORDER BY 4, 1, 2;
+ """.format(case_str , schema.lower()), True)
+
+ drop_str = ""
+
+ for idx in range(len(madlib_functions)):
+
+ func = madlib_functions[idx]
+ # We don't drop type related functions
+ no_drop = ['bytea8', 'float8arr', 'svec']
+ if not any(x in func['name'] for x in no_drop):
+ drop_str = drop_str + "DROP {0} {1}.{2}({3}); \n".format(
+ func['type'], schema, func['name'], func['args'])
+
+ # Break the drop file into chunks because some systems complain with
long sql commands
+ if idx%100 == 99:
+ drop_str = drop_str + "\n"
+ return drop_str
+
+
+def get_madlib_operator_drop_str(schema):
+
+ madlib_operators = _internal_run_query("""
+ SELECT n.nspname as "Schema",
+ o.oprname AS "name",
+ CASE WHEN o.oprkind='l' THEN NULL
+ ELSE pg_catalog.format_type(o.oprleft, NULL) END AS "left_op",
+ CASE WHEN o.oprkind='r' THEN NULL
+ ELSE pg_catalog.format_type(o.oprright, NULL) END AS "right_op"
+ FROM pg_catalog.pg_operator o
+ LEFT JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace
+ WHERE n.nspname OPERATOR(pg_catalog.~) '^(%s)$'
+ ORDER BY 1, 2, 3, 4;
+ """ % schema.lower(), True)
+
+ # Drop the Operator Class before the individual operators
+ drop_str = "DROP OPERATOR CLASS IF EXISTS {0}.svec_ops USING btree;
\n".format(schema)
+
+ for i in madlib_operators:
+ drop_str = drop_str + "DROP OPERATOR {0}.{1}({2},{3}); \n".format(
+ schema, i['name'], i['left_op'], i['right_op'])
+ return drop_str
+
def main(argv):
args = parse_arguments()
@@ -1225,7 +1288,6 @@ def main(argv):
schema = args.schema[0].lower()
else:
schema = args.schema.lower()
-
# Parse DB Platform (== PortID) and compare with Ports.yml
global portid
if args.platform:
@@ -1396,7 +1458,6 @@ def main(argv):
info_(this, "%s MADlib:" % op_msg, True)
_cleanup_comments_in_sqlfile(output_filename, upgrade)
result = _run_sql_file(schema, output_filename)
-
if result == 'FAIL':
info_(this, "MADlib {0}
unsuccessful.".format(args.command[0]), True)
info_(this, "All changes are rolled back.", True)
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index 9197b99..2a44e8b 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -449,14 +449,6 @@ class ChangeHandler(UpgradeBase):
format(sourcetype=self._udc[udc]['sourcetype'],
targettype=self._udc[udc]['targettype']))
- def drop_traininginfo_4dt(self):
- """
- @brief Drop the madlib.training_info table, which should no longer be
used since
- the version 1.5
- """
- _write_to_file(self.output_filehandle, "DROP TABLE IF EXISTS
{schema}.training_info;".
- format(schema=self._schema))
-
def drop_changed_udo(self):
"""
@brief Drop all operators (UDO) that were removed/updated in new
version
@@ -1314,10 +1306,10 @@ class ScriptCleaner(UpgradeBase):
self._clean_comment()
self._clean_type()
self._clean_cast()
- self._clean_operator()
- self._clean_opclass()
- self._clean_aggregate()
- self._clean_function()
+ # self._clean_operator()
+ # self._clean_opclass()
+ # self._clean_aggregate()
+ # self._clean_function()
return self._sql
diff --git a/src/madpack/utilities.py b/src/madpack/utilities.py
index 9e4e95b..be84de2 100644
--- a/src/madpack/utilities.py
+++ b/src/madpack/utilities.py
@@ -28,7 +28,6 @@ import sys
import subprocess
import unittest
-
# Some read-only variables
this = os.path.basename(sys.argv[0]) # name of this script
