This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 4330668 PHOENIX-5656 Make Phoenix scripts work with Python 3
4330668 is described below
commit 4330668523afcb03fc99dfa861607452b8829b9b
Author: Richard Antal <[email protected]>
AuthorDate: Tue May 12 15:41:14 2020 +0200
PHOENIX-5656 Make Phoenix scripts work with Python 3
Co-authored-by: Lars Hofhansl <[email protected]>
---
bin/end2endTest.py | 5 +++--
bin/performance.py | 25 ++++++++++++-----------
bin/pherf-cluster.py | 12 ++++++-----
bin/pherf-standalone.py | 8 +++++---
bin/phoenix_sandbox.py | 5 +++--
bin/phoenix_utils.py | 53 +++++++++++++++++++++++++++++--------------------
bin/psql.py | 8 +++++---
bin/sqlline.py | 18 +++++++++--------
bin/traceserver.py | 38 +++++++++++++++++++----------------
9 files changed, 98 insertions(+), 74 deletions(-)
diff --git a/bin/end2endTest.py b/bin/end2endTest.py
index 32621f6..40954d1 100755
--- a/bin/end2endTest.py
+++ b/bin/end2endTest.py
@@ -23,6 +23,7 @@
# !!! Do NOT run the script against a prodcution cluster because it wipes out
# !!! existing data of the cluster
+from __future__ import print_function
import os
import subprocess
import sys
@@ -36,8 +37,8 @@ phoenix_jar_path =
os.getenv(phoenix_utils.phoenix_class_path, phoenix_utils.pho
# HBase/Phoenix client side property override
hbase_library_path = os.getenv('HBASE_LIBRARY_DIR', '')
-print "Current ClassPath=%s:%s:%s" % (phoenix_utils.hbase_conf_dir,
phoenix_jar_path,
- hbase_library_path)
+print("Current ClassPath=%s:%s:%s" % (phoenix_utils.hbase_conf_dir,
phoenix_jar_path,
+ hbase_library_path))
java_cmd = "java -cp " + phoenix_utils.hbase_conf_dir + os.pathsep +
phoenix_jar_path + os.pathsep + \
hbase_library_path + " org.apache.phoenix.end2end.End2EndTestDriver " + \
diff --git a/bin/performance.py b/bin/performance.py
index f61ad20..16fee48 100755
--- a/bin/performance.py
+++ b/bin/performance.py
@@ -19,6 +19,7 @@
#
############################################################################
+from __future__ import print_function
import os
import subprocess
import sys
@@ -27,7 +28,7 @@ import phoenix_utils
def queryex(description, statement):
global statements
- print "Query # %s - %s" % (description, statement)
+ print("Query # %s - %s" % (description, statement))
statements = statements + statement
def delfile(filename):
@@ -35,9 +36,9 @@ def delfile(filename):
os.remove(filename)
def usage():
- print "Performance script arguments not specified. Usage: performance.py \
-<zookeeper> <row count>"
- print "Example: performance.py localhost 100000"
+ print("Performance script arguments not specified. Usage: performance.py \
+<zookeeper> <row count>")
+ print("Example: performance.py localhost 100000")
def createFileWithContent(filename, content):
@@ -78,17 +79,17 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = x.decode().partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
if java_home:
@@ -110,10 +111,10 @@ SPLIT ON
('CSGoogle','CSSalesforce','EUApple','EUGoogle','EUSalesforce',\
'NAApple','NAGoogle','NASalesforce');" % (table)
# generate and upsert data
-print "Phoenix Performance Evaluation Script 1.0"
-print "-----------------------------------------"
+print("Phoenix Performance Evaluation Script 1.0")
+print("-----------------------------------------")
-print "\nCreating performance table..."
+print("\nCreating performance table...")
createFileWithContent(ddl, createtable)
exitcode = subprocess.call(execute + ddl, shell=True)
@@ -127,13 +128,13 @@ queryex("3 - Group By Second PK", "SELECT DOMAIN FROM %s
GROUP BY DOMAIN;" % (ta
queryex("4 - Truncate + Group By", "SELECT TRUNC(DATE,'DAY') DAY FROM %s GROUP
BY TRUNC(DATE,'DAY');" % (table))
queryex("5 - Filter + Count", "SELECT COUNT(1) FROM %s WHERE CORE<10;" %
(table))
-print "\nGenerating and upserting data..."
+print("\nGenerating and upserting data...")
exitcode = subprocess.call('%s -jar %s %s %s' % (java_cmd,
phoenix_utils.testjar, data, rowcount),
shell=True)
if exitcode != 0:
sys.exit(exitcode)
-print "\n"
+print("\n")
createFileWithContent(qry, statements)
exitcode = subprocess.call(execute + data + ' ' + qry, shell=True)
diff --git a/bin/pherf-cluster.py b/bin/pherf-cluster.py
index 031cfd9..4c688f9 100755
--- a/bin/pherf-cluster.py
+++ b/bin/pherf-cluster.py
@@ -22,6 +22,8 @@
# This script is intended for use where HBase/Phoenix is loaded from HBase
classpath
# therefore HBASE_DIR environment variable needs to be configured for this
script to execute
+from __future__ import print_function
+from phoenix_utils import tryDecode
import os
import subprocess
import sys
@@ -48,17 +50,17 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = tryDecode(x).partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
if java_home:
@@ -66,7 +68,7 @@ if java_home:
else:
java = 'java'
-print "HBASE_DIR environment variable is currently set to: " + hbase_path
+print("HBASE_DIR environment variable is currently set to: " + hbase_path)
# Get the HBase classpath
hbasecp, stderr = subprocess.Popen(hbase_path + "/bin/hbase classpath",
@@ -74,7 +76,7 @@ hbasecp, stderr = subprocess.Popen(hbase_path + "/bin/hbase
classpath",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
-java_cmd = java +' -cp "' + hbasecp + os.pathsep +
phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir +
os.pathsep + phoenix_utils.phoenix_pherf_jar + \
+java_cmd = java +' -cp "' + tryDecode(hbasecp) + os.pathsep +
phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir +
os.pathsep + phoenix_utils.phoenix_pherf_jar + \
'" -Dlog4j.configuration=file:' + \
os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" org.apache.phoenix.pherf.Pherf " + args
diff --git a/bin/pherf-standalone.py b/bin/pherf-standalone.py
index b0d593e..b87585e 100755
--- a/bin/pherf-standalone.py
+++ b/bin/pherf-standalone.py
@@ -19,6 +19,8 @@
#
############################################################################
+from __future__ import print_function
+from phoenix_utils import tryDecode
import os
import subprocess
import sys
@@ -44,17 +46,17 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = tryDecode(x).partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
if java_home:
diff --git a/bin/phoenix_sandbox.py b/bin/phoenix_sandbox.py
index 4279dd6..e18c686 100755
--- a/bin/phoenix_sandbox.py
+++ b/bin/phoenix_sandbox.py
@@ -19,6 +19,7 @@
#
############################################################################
+from __future__ import print_function
import os
import subprocess
import sys
@@ -51,12 +52,12 @@ proc = subprocess.Popen(java_cmd, shell=True)
try:
proc.wait()
except KeyboardInterrupt:
- print "Shutting down sandbox..."
+ print("Shutting down sandbox...")
proc.terminate()
proc.wait()
-print "Sandbox is stopped"
+print("Sandbox is stopped")
returncode = proc.returncode
if returncode is not None:
sys.exit(returncode)
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index bbcab26..424bab0 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -19,6 +19,7 @@
#
############################################################################
+from __future__ import print_function
import os
import fnmatch
import subprocess
@@ -41,6 +42,14 @@ def find(pattern, classPaths):
return ""
+def tryDecode(input):
+ """ Python 2/3 compatibility hack
+ """
+ try:
+ return input.decode()
+ except:
+ return input
+
def findFileInPathWithoutRecursion(pattern, path):
if not os.path.exists(path):
return ""
@@ -65,7 +74,7 @@ def findClasspath(command_name):
# We don't have this command, so we can't get its classpath
return ''
command = "%s%s" %(command_path, ' classpath')
- return subprocess.Popen(command, shell=True,
stdout=subprocess.PIPE).stdout.read()
+ return tryDecode(subprocess.Popen(command, shell=True,
stdout=subprocess.PIPE).stdout.read())
def setPath():
PHOENIX_CLIENT_JAR_PATTERN = "phoenix-client.jar"
@@ -133,18 +142,18 @@ def setPath():
global hadoop_classpath
if (os.name != 'nt'):
- hadoop_classpath = findClasspath('hadoop')
+ hadoop_classpath = findClasspath('hadoop').rstrip()
else:
- hadoop_classpath = os.getenv('HADOOP_CLASSPATH', '')
+ hadoop_classpath = os.getenv('HADOOP_CLASSPATH', '').rstrip()
global hadoop_common_jar_path
- hadoop_common_jar_path = os.path.join(current_dir, "..", "phoenix-client",
"target","*")
+ hadoop_common_jar_path = os.path.join(current_dir, "..", "phoenix-client",
"target","*").rstrip()
global hadoop_common_jar
hadoop_common_jar = find("hadoop-common*.jar", hadoop_common_jar_path)
global hadoop_hdfs_jar_path
- hadoop_hdfs_jar_path = os.path.join(current_dir, "..", "phoenix-client",
"target","*")
+ hadoop_hdfs_jar_path = os.path.join(current_dir, "..", "phoenix-client",
"target","*").rstrip()
global hadoop_hdfs_jar
hadoop_hdfs_jar = find("hadoop-hdfs*.jar", hadoop_hdfs_jar_path)
@@ -204,7 +213,7 @@ def shell_quote(args):
else:
# pipes module isn't available on Windows
import pipes
- return " ".join([pipes.quote(v) for v in args])
+ return " ".join([pipes.quote(tryDecode(v)) for v in args])
def common_sqlline_args(parser):
parser.add_argument('-v', '--verbose', help='Verbosity on sqlline.',
default='true')
@@ -213,19 +222,19 @@ def common_sqlline_args(parser):
if __name__ == "__main__":
setPath()
- print "phoenix_class_path:", phoenix_class_path
- print "hbase_conf_dir:", hbase_conf_dir
- print "hbase_conf_path:", hbase_conf_path
- print "current_dir:", current_dir
- print "phoenix_jar_path:", phoenix_jar_path
- print "phoenix_client_jar:", phoenix_client_jar
- print "phoenix_test_jar_path:", phoenix_test_jar_path
- print "hadoop_common_jar_path:", hadoop_common_jar_path
- print "hadoop_common_jar:", hadoop_common_jar
- print "hadoop_hdfs_jar_path:", hadoop_hdfs_jar_path
- print "hadoop_hdfs_jar:", hadoop_hdfs_jar
- print "testjar:", testjar
- print "phoenix_queryserver_jar:", phoenix_queryserver_jar
- print "phoenix_loadbalancer_jar:", phoenix_loadbalancer_jar
- print "phoenix_thin_client_jar:", phoenix_thin_client_jar
- print "hadoop_classpath:", hadoop_classpath
+ print("phoenix_class_path:", phoenix_class_path)
+ print("hbase_conf_dir:", hbase_conf_dir)
+ print("hbase_conf_path:", hbase_conf_path)
+ print("current_dir:", current_dir)
+ print("phoenix_jar_path:", phoenix_jar_path)
+ print("phoenix_client_jar:", phoenix_client_jar)
+ print("phoenix_test_jar_path:", phoenix_test_jar_path)
+ print("hadoop_common_jar_path:", hadoop_common_jar_path)
+ print("hadoop_common_jar:", hadoop_common_jar)
+ print("hadoop_hdfs_jar_path:", hadoop_hdfs_jar_path)
+ print("hadoop_hdfs_jar:", hadoop_hdfs_jar)
+ print("testjar:", testjar)
+ print("phoenix_queryserver_jar:", phoenix_queryserver_jar)
+ print("phoenix_loadbalancer_jar:", phoenix_loadbalancer_jar)
+ print("phoenix_thin_client_jar:", phoenix_thin_client_jar)
+ print("hadoop_classpath:", hadoop_classpath)
diff --git a/bin/psql.py b/bin/psql.py
index 2002cb3..0e57c77 100755
--- a/bin/psql.py
+++ b/bin/psql.py
@@ -19,6 +19,8 @@
#
############################################################################
+from __future__ import print_function
+from phoenix_utils import tryDecode
import os
import subprocess
import sys
@@ -44,17 +46,17 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = tryDecode(x).partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
if java_home:
diff --git a/bin/sqlline.py b/bin/sqlline.py
index 00c2d92..23e54c5 100755
--- a/bin/sqlline.py
+++ b/bin/sqlline.py
@@ -19,6 +19,8 @@
#
############################################################################
+from __future__ import print_function
+from phoenix_utils import tryDecode
import os
import subprocess
import sys
@@ -56,8 +58,8 @@ phoenix_utils.common_sqlline_args(parser)
# Parse the args
args=parser.parse_args()
-zookeeper = args.zookeepers
-sqlfile = args.sqlfile
+zookeeper = tryDecode(args.zookeepers)
+sqlfile = tryDecode(args.sqlfile)
# HBase configuration folder path (where hbase-site.xml reside) for
# HBase/Phoenix client side property override
@@ -82,17 +84,17 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = x.decode().partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
if java_home:
@@ -100,7 +102,7 @@ if java_home:
else:
java = 'java'
-colorSetting = args.color
+colorSetting = tryDecode(args.color)
# disable color setting for windows OS
if os.name == 'nt':
colorSetting = "false"
@@ -112,7 +114,7 @@ java_cmd = java + ' $PHOENIX_OPTS ' + \
os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver" + \
" -u jdbc:phoenix:" + phoenix_utils.shell_quote([zookeeper]) + \
- " -n none -p none --color=" + colorSetting + " --fastConnect=" +
args.fastconnect + \
- " --verbose=" + args.verbose + " --incremental=false
--isolation=TRANSACTION_READ_COMMITTED " + sqlfile
+ " -n none -p none --color=" + colorSetting + " --fastConnect=" +
tryDecode(args.fastconnect) + \
+ " --verbose=" + tryDecode(args.verbose) + " --incremental=false
--isolation=TRANSACTION_READ_COMMITTED " + sqlfile
os.execl("/bin/sh", "/bin/sh", "-c", java_cmd)
diff --git a/bin/traceserver.py b/bin/traceserver.py
index 356009c..35a918c 100755
--- a/bin/traceserver.py
+++ b/bin/traceserver.py
@@ -25,6 +25,8 @@
# usage: traceserver.py [start|stop]
#
+from __future__ import print_function
+from phoenix_utils import tryDecode
import datetime
import getpass
import os
@@ -49,9 +51,9 @@ command = None
args = sys.argv
if len(args) > 1:
- if args[1] == 'start':
+ if tryDecode(args[1]) == 'start':
command = 'start'
- elif args[1] == 'stop':
+ elif tryDecode(args[1]) == 'stop':
command = 'stop'
if command:
args = args[2:]
@@ -60,7 +62,7 @@ if os.name == 'nt':
args = subprocess.list2cmdline(args[1:])
else:
import pipes # pipes module isn't available on Windows
- args = " ".join([pipes.quote(v) for v in args[1:]])
+ args = " ".join([pipes.quote(tryDecode(v)) for v in args[1:]])
# HBase configuration folder path (where hbase-site.xml reside) for
# HBase/Phoenix client side property override
@@ -86,23 +88,23 @@ elif os.name == 'nt':
hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
if not hbase_env_path or not hbase_env_cmd:
- print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name,
os.linesep))
sys.exit(-1)
hbase_env = {}
if os.path.isfile(hbase_env_path):
p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
for x in p.stdout:
- (k, _, v) = x.partition('=')
+ (k, _, v) = tryDecode(x).partition('=')
hbase_env[k.strip()] = v.strip()
-if hbase_env.has_key('JAVA_HOME'):
+if 'JAVA_HOME' in hbase_env:
java_home = hbase_env['JAVA_HOME']
-if hbase_env.has_key('HBASE_PID_DIR'):
+if 'HBASE_PID_DIR' in hbase_env:
hbase_pid_dir = hbase_env['HBASE_PID_DIR']
-if hbase_env.has_key('HBASE_LOG_DIR'):
+if 'HBASE_LOG_DIR' in hbase_env:
phoenix_log_dir = hbase_env['HBASE_LOG_DIR']
-if hbase_env.has_key('PHOENIX_TRACESERVER_OPTS'):
+if 'PHOENIX_TRACESERVER_OPTS' in hbase_env:
opts = hbase_env['PHOENIX_TRACESERVER_OPTS']
log_file_path = os.path.join(phoenix_log_dir, phoenix_log_file)
@@ -130,7 +132,7 @@ java_cmd = '%(java)s ' + \
if command == 'start':
if not daemon_supported:
- print >> sys.stderr, "daemon mode not supported on this platform"
+ sys.stderr.write("daemon mode not supported on this
platform{}".format(os.linesep))
sys.exit(-1)
# run in the background
@@ -143,7 +145,7 @@ if command == 'start':
stdout = out,
stderr = out,
)
- print 'starting Trace Server, logging to %s' % log_file_path
+ print('starting Trace Server, logging to %s' % log_file_path)
with context:
# this block is the main() for the forked daemon process
child = None
@@ -156,21 +158,23 @@ if command == 'start':
sys.exit(0)
signal.signal(signal.SIGTERM, handler)
- print '%s launching %s' % (datetime.datetime.now(), cmd)
+ print('%s launching %s' % (datetime.datetime.now(), cmd))
child = subprocess.Popen(cmd.split())
sys.exit(child.wait())
elif command == 'stop':
if not daemon_supported:
- print >> sys.stderr, "daemon mode not supported on this platform"
+ sys.stderr.write("daemon mode not supported on this
platform{}".format(os.linesep))
sys.exit(-1)
if not os.path.exists(pid_file_path):
- print >> sys.stderr, "no Trace Server to stop because PID file not
found, %s" % pid_file_path
+ sys.stderr.write("no Trace Server to stop because PID file not found,
{}{}"
+ .format(pid_file_path, os.linesep))
sys.exit(0)
if not os.path.isfile(pid_file_path):
- print >> sys.stderr, "PID path exists but is not a file! %s" %
pid_file_path
+ sys.stderr.write("PID path exists but is not a file! {}{}"
+ .format(pid_file_path, os.linesep))
sys.exit(1)
pid = None
@@ -179,9 +183,9 @@ elif command == 'stop':
if not pid:
sys.exit("cannot read PID file, %s" % pid_file_path)
- print "stopping Trace Server pid %s" % pid
+ print("stopping Trace Server pid %s" % pid)
with open(out_file_path, 'a+') as out:
- print >> out, "%s terminating Trace Server" % datetime.datetime.now()
+ out.write("%s terminating Trace Server%s" % (datetime.datetime.now(),
os.linesep))
os.kill(pid, signal.SIGTERM)
else:
