Github user njayaram2 commented on a diff in the pull request:
https://github.com/apache/madlib/pull/271#discussion_r192205268
--- Diff: src/madpack/madpack.py ---
@@ -987,275 +1276,42 @@ def main(argv):
error_(this, "Missing -p/--platform parameter.", True)
if not con_args:
error_(this, "Unknown problem with database connection string:
%s" % con_args, True)
+ # ---------------- Completed "Get and validate arguments"
-----------------
# COMMAND: version
if args.command[0] == 'version':
_print_revs(rev, dbrev, con_args, schema)
- # COMMAND: uninstall/reinstall
- if args.command[0] in ('uninstall', 'reinstall'):
- if get_rev_num(dbrev) == [0]:
- info_(this, "Nothing to uninstall. No version found in schema
%s." % schema.upper(), True)
- return
-
- # Find any potential data to lose
- affected_objects = _internal_run_query("""
- SELECT
- n1.nspname AS schema,
- relname AS relation,
- attname AS column,
- typname AS type
- FROM
- pg_attribute a,
- pg_class c,
- pg_type t,
- pg_namespace n,
- pg_namespace n1
- WHERE
- n.nspname = '%s'
- AND t.typnamespace = n.oid
- AND a.atttypid = t.oid
- AND c.oid = a.attrelid
- AND c.relnamespace = n1.oid
- AND c.relkind = 'r'
- ORDER BY
- n1.nspname, relname, attname, typname""" % schema.lower(),
True)
-
- info_(this, "*** Uninstalling MADlib ***", True)
- info_(this,
"***********************************************************************************",
True)
- info_(this, "* Schema %s and all database objects depending on it
will be dropped!" % schema.upper(), True)
- if affected_objects:
- info_(this, "* If you continue the following data will be lost
(schema : table.column : type):", True)
- for ao in affected_objects:
- info_(this, '* - ' + ao['schema'] + ' : ' + ao['relation']
+ '.' +
- ao['column'] + ' : ' + ao['type'], True)
- info_(this,
"***********************************************************************************",
True)
- info_(this, "Would you like to continue? [Y/N]", True)
- go = raw_input('>>> ').upper()
- while go != 'Y' and go != 'N':
- go = raw_input('Yes or No >>> ').upper()
-
- # 2) Do the uninstall/drop
- if go == 'N':
- info_(this, 'No problem. Nothing dropped.', True)
- return
-
- elif go == 'Y':
- info_(this, "> dropping schema %s" % schema.upper(), verbose)
- try:
- _internal_run_query("DROP SCHEMA %s CASCADE;" % (schema),
True)
- except:
- error_(this, "Cannot drop schema %s." % schema.upper(),
True)
-
- info_(this, 'Schema %s (and all dependent objects) has been
dropped.' % schema.upper(), True)
- info_(this, 'MADlib uninstalled successfully.', True)
-
- else:
- return
-
- # COMMAND: install/reinstall
- if args.command[0] in ('install', 'reinstall'):
- # Refresh MADlib version in DB, None for GP/PG
- if args.command[0] == 'reinstall':
- print "Setting MADlib database version to be None for
reinstall"
- dbrev = None
-
- info_(this, "*** Installing MADlib ***", True)
-
- # 1) Compare OS and DB versions.
- # noop if OS <= DB.
- _print_revs(rev, dbrev, con_args, schema)
- if is_rev_gte(get_rev_num(dbrev), get_rev_num(rev)):
- info_(this, "Current MADlib version already up to date.", True)
- return
- # proceed to create objects if nothing installed in DB
- elif dbrev is None:
- pass
- # error and refer to upgrade if OS > DB
- else:
- error_(this, """Aborting installation: existing MADlib version
detected in {0} schema
- To upgrade the {0} schema to MADlib v{1} please run
the following command:
- madpack upgrade -s {0} -p {2} [-c ...]
- """.format(schema, rev, portid), True)
-
- # 2) Run installation
- try:
- _plpy_check(py_min_ver)
- _db_install(schema, dbrev, args.testcase)
- except:
- error_(this, "MADlib installation failed.", True)
-
- # COMMAND: upgrade
- if args.command[0] in ('upgrade', 'update'):
- info_(this, "*** Upgrading MADlib ***", True)
- dbrev = get_madlib_dbrev(con_args, schema)
-
- # 1) Check DB version. If None, nothing to upgrade.
- if not dbrev:
- info_(this, "MADlib is not installed in {schema} schema and
there "
- "is nothing to upgrade. Please use install "
- "instead.".format(schema=schema.upper()),
- True)
- return
-
- # 2) Compare OS and DB versions. Continue if OS > DB.
- _print_revs(rev, dbrev, con_args, schema)
- if is_rev_gte(get_rev_num(dbrev), get_rev_num(rev)):
- info_(this, "Current MADlib version is already up-to-date.",
True)
- return
-
- if float('.'.join(dbrev.split('.')[0:2])) < 1.0:
- info_(this, "The version gap is too large, upgrade is
supported only for "
- "packages greater than or equal to v1.0.", True)
- return
-
- # 3) Run upgrade
- try:
- _plpy_check(py_min_ver)
- _db_upgrade(schema, dbrev)
- except Exception as e:
- # Uncomment the following lines when debugging
- print "Exception: " + str(e)
- print sys.exc_info()
- traceback.print_tb(sys.exc_info()[2])
- error_(this, "MADlib upgrade failed.", True)
-
# COMMAND: install-check
if args.command[0] == 'install-check':
-
- # 1) Compare OS and DB versions. Continue if OS = DB.
- if get_rev_num(dbrev) != get_rev_num(rev):
- _print_revs(rev, dbrev, con_args, schema)
- info_(this, "Versions do not match. Install-check stopped.",
True)
- return
-
- # Create install-check user
- test_user = ('madlib_' +
- rev.replace('.', '').replace('-', '_') +
- '_installcheck')
+ run_install_check(locals(), args.testcase)
+ else:
try:
- _internal_run_query("DROP USER IF EXISTS %s;" % (test_user),
False)
+ is_schema_in_db = _internal_run_query("SELECT schema_name FROM
information_schema.schemata WHERE schema_name='%s';" % schema, True)
except:
- _internal_run_query("DROP OWNED BY %s CASCADE;" % (test_user),
True)
- _internal_run_query("DROP USER IF EXISTS %s;" % (test_user),
True)
- _internal_run_query("CREATE USER %s;" % (test_user), True)
-
- _internal_run_query("GRANT USAGE ON SCHEMA %s TO %s;" % (schema,
test_user), True)
-
- # 2) Run test SQLs
- info_(this, "> Running test scripts for:", verbose)
-
- caseset = (set([test.strip() for test in args.testcase.split(',')])
- if args.testcase != "" else set())
-
- modset = {}
- for case in caseset:
- if case.find('/') > -1:
- [mod, algo] = case.split('/')
- if mod not in modset:
- modset[mod] = []
- if algo not in modset[mod]:
- modset[mod].append(algo)
- else:
- modset[case] = []
-
- # Loop through all modules
- for moduleinfo in portspecs['modules']:
-
- # Get module name
- module = moduleinfo['name']
-
- # Skip if doesn't meet specified modules
- if modset is not None and len(modset) > 0 and module not in
modset:
- continue
- # JIRA: MADLIB-1078 fix
- # Skip pmml during install-check (when run without the -t
option).
- # We can still run install-check on pmml with '-t' option.
- if not modset and module in ['pmml']:
- continue
- info_(this, "> - %s" % module, verbose)
-
- # Make a temp dir for this module (if doesn't exist)
- cur_tmpdir = tmpdir + '/' + module + '/test' # tmpdir is a
global variable
- _make_dir(cur_tmpdir)
-
- # Find the Python module dir (platform specific or generic)
- if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver +
"/modules/" + module):
- maddir_mod_py = maddir + "/ports/" + portid + "/" + dbver
+ "/modules"
+ error_(this, "Cannot validate if schema already exists.", True)
+
+ output_filename = tmpdir +
"/madlib_{0}.sql".format(args.command[0])
+ upgrade = False
+ return_val = create_install_madlib_sqlfile(locals(),
args.command[0], args.testcase)
+ if return_val == 0:
+ info_(this, "Installing MADlib modules...", True)
+ _cleanup_comments_in_sqlfile(output_filename, upgrade)
+ result = _run_sql_file(schema, output_filename)
+
+ if result == 'FAIL':
+ info_(this, "MADlib {0}
unsuccessful.".format(args.command[0]), True)
else:
- maddir_mod_py = maddir + "/modules"
-
- # Find the SQL module dir (platform specific or generic)
- if os.path.isdir(maddir + "/ports/" + portid + "/modules/" +
module):
- maddir_mod_sql = maddir + "/ports/" + portid + "/modules"
- else:
- maddir_mod_sql = maddir + "/modules"
-
- # Prepare test schema
- test_schema = "madlib_installcheck_%s" % (module)
- _internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE; CREATE
SCHEMA %s;" %
- (test_schema, test_schema), True)
- _internal_run_query("GRANT ALL ON SCHEMA %s TO %s;" %
- (test_schema, test_user), True)
-
- # Switch to test user and prepare the search_path
- pre_sql = '-- Switch to test user:\n' \
- 'SET ROLE %s;\n' \
- '-- Set SEARCH_PATH for install-check:\n' \
- 'SET search_path=%s,%s;\n' \
- % (test_user, test_schema, schema)
-
- # Loop through all test SQL files for this module
- sql_files = maddir_mod_sql + '/' + module + '/test/*.sql_in'
- for sqlfile in sorted(glob.glob(sql_files), reverse=True):
- algoname = os.path.basename(sqlfile).split('.')[0]
- # run only algo specified
- if (module in modset and modset[module] and
- algoname not in modset[module]):
- continue
-
- # Set file names
- tmpfile = cur_tmpdir + '/' + os.path.basename(sqlfile) +
'.tmp'
- logfile = cur_tmpdir + '/' + os.path.basename(sqlfile) +
'.log'
-
- # If there is no problem with the SQL file
- milliseconds = 0
-
- # Run the SQL
- run_start = datetime.datetime.now()
- retval = _run_sql_file(schema, maddir_mod_py, module,
- sqlfile, tmpfile, logfile, pre_sql)
- # Runtime evaluation
- run_end = datetime.datetime.now()
- milliseconds = round((run_end - run_start).seconds * 1000 +
- (run_end - run_start).microseconds /
1000)
-
- # Check the exit status
- if retval != 0:
- result = 'FAIL'
- keeplogs = True
- # Since every single statement in the test file gets
logged,
- # an empty log file indicates an empty or a failed test
- elif os.path.isfile(logfile) and os.path.getsize(logfile)
> 0:
- result = 'PASS'
- # Otherwise
- else:
- result = 'ERROR'
-
- # Output result
- print "TEST CASE RESULT|Module: " + module + \
- "|" + os.path.basename(sqlfile) + "|" + result + \
- "|Time: %d milliseconds" % (milliseconds)
-
- if result == 'FAIL':
- error_(this, "Failed executing %s" % tmpfile, False)
- error_(this, "Check the log at %s" % logfile, False)
- # Cleanup test schema for the module
- _internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE;" %
(test_schema), True)
-
- # Drop install-check user
- _internal_run_query("DROP OWNED BY %s CASCADE;" % (test_user),
True)
- _internal_run_query("DROP USER %s;" % (test_user), True)
+ if args.command[0] != 'uninstall':
+ if args.command[0] == 'upgrade':
+ info_(this, "MADlib %s upgraded successfully in %s
schema." % (str(rev), schema.upper()), True)
+ else:
+ info_(this, "> Created %s schema" %
schema.upper(), True)
--- End diff --
Removed `upper()` function calls.
---
