This is an automated email from the ASF dual-hosted git repository.
jiaqizho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit f9f8bdbb2d1bf8f0f53fd7c2a8bdc42c72c15ae4
Author: Anusha Shakarad <shakar...@vmware.com>
AuthorDate: Wed Dec 21 15:49:12 2022 +0800
Update scripts to use python3
For gpdb7 will only support python3 and we also update the
gpMgmt/requirements-dev.txt to remove the stick the modules to the python2
compatible versions introduced by the commit 68a8237.
[GPR-1153]
Co-authored-by: Anusha Shakarad <shakar...@vmware.com>
Co-authored-by: Ning Wu <ni...@vmware.com>
---
concourse/scripts/builds/GpBuild.py | 8 +++---
concourse/scripts/perfsummary.py | 16 ++++++------
concourse/scripts/regression_tests_gpcloud.bash | 4 +--
concourse/scripts/unit_tests_gporca.bash | 2 +-
.../bin/gppylib/test/unit/test_unit_gpcheckcat.py | 6 ++---
gpMgmt/requirements-dev.txt | 3 ++-
gpcontrib/gpcloud/bin/dummyHTTPServer.py | 30 +++++++++++-----------
.../gporca/concourse/xerces-c/build_xerces.py | 10 ++++----
src/test/locale/sort-test.py | 2 +-
src/test/performance/parse_perf_results.py | 2 +-
10 files changed, 42 insertions(+), 41 deletions(-)
diff --git a/concourse/scripts/builds/GpBuild.py
b/concourse/scripts/builds/GpBuild.py
index 0eb041bfac..2ed28467d5 100755
--- a/concourse/scripts/builds/GpBuild.py
+++ b/concourse/scripts/builds/GpBuild.py
@@ -52,7 +52,7 @@ class GpBuild:
cmd = source_env_cmd
runcmd = "runuser gpadmin -c \"{0} && {1} \"".format(cmd, command)
if print_command:
- print "Executing {}".format(runcmd)
+ print("Executing {}".format(runcmd))
return subprocess.call([runcmd], shell=True, stdout=stdout,
stderr=stderr)
def run_explain_test_suite(self, dbexists):
@@ -74,7 +74,7 @@ class GpBuild:
status = self._run_gpdb_command("psql -q -f stats.sql",
stdout=f)
if status:
with open("load_stats.txt", "r") as f:
- print f.read()
+ print(f.read())
fail_on_error(status)
# set gucs if any were specified
@@ -95,10 +95,10 @@ class GpBuild:
if fsql.endswith('.sql') and fsql not in ['stats.sql',
'schema.sql']:
output_fname = 'out/{}'.format(fsql.replace('.sql', '.out'))
with open(output_fname, 'w') as fout:
- print "Running query: " + fsql
+ print("Running query: " + fsql)
current_status = self._run_gpdb_command("psql -a -f
sql/{}".format(fsql), stdout=fout, stderr=fout, source_env_cmd=source_env_cmd,
print_command=False)
if current_status != 0:
- print "ERROR: {0}".format(current_status)
+ print("ERROR: {0}".format(current_status))
status = status if status != 0 else current_status
return status
diff --git a/concourse/scripts/perfsummary.py b/concourse/scripts/perfsummary.py
index 0a8bfae0db..cab68c1408 100755
--- a/concourse/scripts/perfsummary.py
+++ b/concourse/scripts/perfsummary.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# Extracts summary info from a log file that compiles and executes test suite
# queries. Creates a CSV (comma-separated values) summary.
@@ -209,20 +209,20 @@ class FileState:
# print header for CSV file
def printHeader(self, numFiles):
if (numFiles == 1):
- print 'Query id, planning time, execution time, comment'
+ print('Query id, planning time, execution time, comment')
else:
- print 'Query id, base planning time, planning time, base execution
time, execution time, plan changes, base comment, comment'
+ print('Query id, base planning time, planning time, base execution
time, execution time, plan changes, base comment, comment')
# print result for all recorded queries in CSV format for a single log file
def printme(self):
for q in self.query_id_list:
- print "%s, %s, %s, %s" % (q, self.query_explain_time_map[q],
self.query_exe_time_map[q], self.query_comment_map[q])
+ print("%s, %s, %s, %s" % (q, self.query_explain_time_map[q],
self.query_exe_time_map[q], self.query_comment_map[q]))
# print a CSV file with a comparison between a base file and a test file
def printComparison(self, base, diffDir, diffThreshold, diffLevel):
for q in self.query_id_list:
planDiffs = self.comparePlans(base, q)
- print "%s, %s, %s, %s, %s, %s, %s, %s" % (q,
base.query_explain_time_map[q], self.query_explain_time_map[q],
base.query_exe_time_map[q], self.query_exe_time_map[q],
planDiffText[planDiffs], base.query_comment_map[q], self.query_comment_map[q])
+ print("%s, %s, %s, %s, %s, %s, %s, %s" % (q,
base.query_explain_time_map[q], self.query_explain_time_map[q],
base.query_exe_time_map[q], self.query_exe_time_map[q],
planDiffText[planDiffs], base.query_comment_map[q], self.query_comment_map[q]))
if int(diffLevel) <= int(planDiffs):
baseTime = float(base.query_exe_time_map[q])
testTime = float(self.query_exe_time_map[q])
@@ -267,7 +267,7 @@ def main():
os.mkdir(diffDir + "/base")
os.mkdir(diffDir + "/test")
except:
- print "Unable to create diff directory %s" % diffDir
+ print("Unable to create diff directory %s" % diffDir)
exit(1)
if args.diffThreshold is not None:
diffThreshold = args.diffThreshold
@@ -278,11 +278,11 @@ def main():
diffLevel = args.diffLevel
else:
if (args.diffThreshold is not None or args.diffLevel is not None):
- print "Please specify the --diffDir option with a directory name
to request diff files\n"
+ print("Please specify the --diffDir option with a directory name
to request diff files\n")
exit(1)
if inputfile is None:
- print "Expected the name of a log file with test suite queries\n"
+ print("Expected the name of a log file with test suite queries\n")
exit(1)
if basefile is not None:
diff --git a/concourse/scripts/regression_tests_gpcloud.bash
b/concourse/scripts/regression_tests_gpcloud.bash
index 5fdc9a1c08..7d3011ad6d 100755
--- a/concourse/scripts/regression_tests_gpcloud.bash
+++ b/concourse/scripts/regression_tests_gpcloud.bash
@@ -37,11 +37,11 @@ function setup_gpadmin_user() {
}
function make_cluster() {
- PYTHONPATH=${SCRIPT_DIR}:${PYTHONPATH} python2 -c "from builds.GpBuild
import GpBuild;
GpBuild(\"planner\").create_demo_cluster(install_dir='$GPDB_INSTALL_DIR')"
+ PYTHONPATH=${SCRIPT_DIR}:${PYTHONPATH} python3 -c "from builds.GpBuild
import GpBuild;
GpBuild(\"planner\").create_demo_cluster(install_dir='$GPDB_INSTALL_DIR')"
}
function configure_with_planner() {
- PYTHONPATH=${SCRIPT_DIR}:${PYTHONPATH} python2 -c "from builds.GpBuild
import GpBuild; GpBuild(\"planner\").configure()"
+ PYTHONPATH=${SCRIPT_DIR}:${PYTHONPATH} python3 -c "from builds.GpBuild
import GpBuild; GpBuild(\"planner\").configure()"
}
function copy_gpdb_bits_to_gphome() {
diff --git a/concourse/scripts/unit_tests_gporca.bash
b/concourse/scripts/unit_tests_gporca.bash
index 458185e5be..c7b6956cdd 100755
--- a/concourse/scripts/unit_tests_gporca.bash
+++ b/concourse/scripts/unit_tests_gporca.bash
@@ -8,7 +8,7 @@ function build_xerces
OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
mkdir -p xerces_patch/concourse
cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse
- /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py
--output_dir=${OUTPUT_DIR}
+ /usr/bin/python3 xerces_patch/concourse/xerces-c/build_xerces.py
--output_dir=${OUTPUT_DIR}
rm -rf build
}
diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py
b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py
index 63f2d88fe8..ccdfb03a7a 100755
--- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py
+++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py
@@ -320,7 +320,7 @@ class GpCheckCatTestCase(GpTestCase):
testargs = ['gpcheckcat', '-port 1', '-s test2']
with patch.object(sys, 'argv', testargs):
self.subject.main()
- mock_run.assert_has_calls(call(['test1', 'test3']))
+ mock_run.assert_has_calls([call(['test1', 'test3'])])
@patch('gpcheckcat.GPCatalog', return_value=Mock())
@patch('sys.exit')
@@ -336,7 +336,7 @@ class GpCheckCatTestCase(GpTestCase):
testargs = ['gpcheckcat', '-port 1', '-s', "test1, test2"]
with patch.object(sys, 'argv', testargs):
self.subject.main()
- mock_run.assert_has_calls(call(['test3']))
+ mock_run.assert_has_calls([call(['test3'])])
@patch('gpcheckcat.GPCatalog', return_value=Mock())
@patch('sys.exit')
@@ -352,7 +352,7 @@ class GpCheckCatTestCase(GpTestCase):
testargs = ['gpcheckcat', '-port 1', '-s', "test_invalid, test2"]
with patch.object(sys, 'argv', testargs):
self.subject.main()
- mock_run.assert_has_calls(call(['test1', 'test3']))
+ mock_run.assert_has_calls([call(['test1', 'test3'])])
expected_message = "'test_invalid' is not a valid test"
log_messages = [args[0][1] for args in
self.subject.logger.log.call_args_list]
self.assertIn(expected_message, log_messages)
diff --git a/gpMgmt/requirements-dev.txt b/gpMgmt/requirements-dev.txt
index c7f3ee5ebe..384167ab49 100644
--- a/gpMgmt/requirements-dev.txt
+++ b/gpMgmt/requirements-dev.txt
@@ -1,4 +1,4 @@
-gsutil<=4.47
+gsutil
behave~=1.2.6
coverage~=4.5
more-itertools<8.1
@@ -37,3 +37,4 @@ pathlib2<=2.3.5
cffi<=1.13.2
scandir<=1.10.0
pycparser<=2.19
+mock<=5.0.0
diff --git a/gpcontrib/gpcloud/bin/dummyHTTPServer.py
b/gpcontrib/gpcloud/bin/dummyHTTPServer.py
index e0c55887ff..8c288e173c 100755
--- a/gpcontrib/gpcloud/bin/dummyHTTPServer.py
+++ b/gpcontrib/gpcloud/bin/dummyHTTPServer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
"""
Very simple HTTP server in python.
@@ -12,8 +12,8 @@ Send a POST request::
curl -d "foo=bar&bin=baz" http://localhost
"""
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-import SocketServer
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import socketserver
import getopt
import sys
import os
@@ -46,8 +46,8 @@ class S(BaseHTTPRequestHandler):
def do_PUT(self):
# Just bounce the request back
- print "----- SOMETHING WAS PUT ------"
- print self.headers
+ print("----- SOMETHING WAS PUT ------")
+ print(self.headers)
length = int(self.headers['Content-Length'])
content = self.rfile.read(length)
self._set_headers(length)
@@ -55,8 +55,8 @@ class S(BaseHTTPRequestHandler):
def do_POST(self):
# Just bounce the request back
- print "----- SOMETHING WAS POST ------"
- print self.headers
+ print("----- SOMETHING WAS POST ------")
+ print(self.headers)
length = int(self.headers['Content-Length'])
content = self.rfile.read(length)
self._set_headers(length)
@@ -64,8 +64,8 @@ class S(BaseHTTPRequestHandler):
def do_DELETE(self):
# Just bounce the request back
- print "----- SOMETHING WAS DELETED ------"
- print self.headers
+ print("----- SOMETHING WAS DELETED ------")
+ print(self.headers)
length = int(self.headers['Content-Length'])
# content = self.rfile.read(length)
self._set_headers(length)
@@ -85,8 +85,8 @@ class PARAM_S(BaseHTTPRequestHandler):
with open(filename, 'r') as f:
content = f.read()
except Exception:
- print "Can not open file:%s" % filename
- return content
+ print("Can not open file:%s" % filename)
+ return content.encode('utf8')
def do_GET(self):
try:
@@ -97,7 +97,7 @@ class PARAM_S(BaseHTTPRequestHandler):
else:
self._set_headers(0)
except KeyError:
- print "Header missing S3_Param_Req"
+ print("Header missing S3_Param_Req")
self._set_headers(0)
def run(server_class=HTTPServer, handler_class=S, port=8553, https=False):
@@ -117,7 +117,7 @@ def run(server_class=HTTPServer, handler_class=S,
port=8553, https=False):
httpd.socket = ssl.wrap_socket (httpd.socket,
keyfile=keyfile,
certfile=certfile, server_side=True)
- print 'Starting http server...'
+ print('Starting http server...')
httpd.serve_forever()
if __name__ == "__main__":
@@ -128,11 +128,11 @@ if __name__ == "__main__":
try:
opts, args =
getopt.getopt(argv[1:],"hsp:f:t:",["--port=","--filename=", "--type="])
except getopt.GetoptError:
- print help_msg
+ print(help_msg)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
- print help_msg
+ print(help_msg)
sys.exit(0)
elif opt == '-s':
use_ssl = True
diff --git a/src/backend/gporca/concourse/xerces-c/build_xerces.py
b/src/backend/gporca/concourse/xerces-c/build_xerces.py
index 7a4fbd1001..8b49e13ac2 100644
--- a/src/backend/gporca/concourse/xerces-c/build_xerces.py
+++ b/src/backend/gporca/concourse/xerces-c/build_xerces.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
import optparse
import os
@@ -6,7 +6,7 @@ import os.path
import subprocess
import sys
import tarfile
-import urllib2
+import urllib.request, urllib.error, urllib.parse
import hashlib
XERCES_SOURCE_URL =
"http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.2.tar.gz";
@@ -29,8 +29,8 @@ def num_cpus():
return 2
def get_xerces_source():
- remote_src = urllib2.urlopen(XERCES_SOURCE_URL)
- local_src = open("xerces_src.tar.gz", "w")
+ remote_src = urllib.request.urlopen(XERCES_SOURCE_URL)
+ local_src = open("xerces_src.tar.gz", "wb")
local_src.write(remote_src.read())
local_src.close()
file_hash =
hashlib.sha256(open('xerces_src.tar.gz','rb').read()).hexdigest()
@@ -73,7 +73,7 @@ def main():
parser.add_option("--output_dir", dest="output_dir", default="install")
(options, args) = parser.parse_args()
if len(args) > 0:
- print "Unknown arguments"
+ print("Unknown arguments")
return 1
status = get_xerces_source()
if status:
diff --git a/src/test/locale/sort-test.py b/src/test/locale/sort-test.py
index 53019038ab..e2f6828db0 100755
--- a/src/test/locale/sort-test.py
+++ b/src/test/locale/sort-test.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
import sys, string, locale
locale.setlocale(locale.LC_ALL, "")
diff --git a/src/test/performance/parse_perf_results.py
b/src/test/performance/parse_perf_results.py
index fd2497e64c..9ab429b0cb 100644
--- a/src/test/performance/parse_perf_results.py
+++ b/src/test/performance/parse_perf_results.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
'''
Parse the results of the performance test into a CSV for loading into
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cloudberry.apache.org
For additional commands, e-mail: commits-h...@cloudberry.apache.org
