Repository: phoenix Updated Branches: refs/heads/master db8ef6816 -> 6a37307c2
PHOENIX-2365 Remove Pherf uber jar and create python scripts to execute from Phoenix bin directory Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/6a37307c Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/6a37307c Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/6a37307c Branch: refs/heads/master Commit: 6a37307c2228044cda62516af5cbfb91346bcecb Parents: db8ef68 Author: Mujtaba <mujt...@apache.org> Authored: Mon Nov 16 11:52:59 2015 -0800 Committer: Mujtaba <mujt...@apache.org> Committed: Mon Nov 16 11:52:59 2015 -0800 ---------------------------------------------------------------------- bin/pherf-cluster.py | 83 ++++++++++++++++++++ bin/pherf-standalone.py | 71 +++++++++++++++++ bin/phoenix_utils.py | 13 +++ dev/make_rc.sh | 6 +- phoenix-pherf/README.md | 2 +- phoenix-pherf/cluster/pherf.sh | 33 -------- phoenix-pherf/config/pherf.properties | 2 +- phoenix-pherf/pom.xml | 9 +-- .../apache/phoenix/pherf/PherfConstants.java | 2 +- .../phoenix/pherf/rules/RulesApplier.java | 2 +- phoenix-pherf/standalone/pherf.sh | 28 ------- 11 files changed, 177 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/bin/pherf-cluster.py ---------------------------------------------------------------------- diff --git a/bin/pherf-cluster.py b/bin/pherf-cluster.py new file mode 100755 index 0000000..37f29a8 --- /dev/null +++ b/bin/pherf-cluster.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +# This script is intended for use where HBase/Phoenix is loaded from HBase classpath +# therefore HBASE_DIR environment variable needs to be configured for this script to execute + +import os +import subprocess +import sys +import phoenix_utils + +phoenix_utils.setPath() + +args = phoenix_utils.shell_quote(sys.argv[1:]) + +# HBase configuration folder path (where hbase-site.xml reside) for +# HBase/Phoenix client side property override +hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir) +hbase_path = os.getenv('HBASE_DIR') + +java_home = os.getenv('JAVA_HOME') + +# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR +hbase_env_path = None +hbase_env_cmd = None +if os.name == 'posix': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh') + hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path] +elif os.name == 'nt': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd') + hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path] +if not hbase_env_path or not hbase_env_cmd: + print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name + sys.exit(-1) + +hbase_env = {} +if os.path.isfile(hbase_env_path): + p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE) + for x in p.stdout: + (k, _, v) = x.partition('=') + hbase_env[k.strip()] = v.strip() + +if hbase_env.has_key('JAVA_HOME'): + java_home = hbase_env['JAVA_HOME'] + +if java_home: + java = os.path.join(java_home, 'bin', 'java') +else: + java = 'java' + +print "HBASE_DIR environment variable is currently set to: " + hbase_path + +# Get the HBase classpath +hbasecp, stderr = subprocess.Popen(hbase_path + "/bin/hbase classpath", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + +java_cmd = java +' -cp "' + hbasecp + os.pathsep + phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_pherf_jar + \ + '" -Dlog4j.configuration=file:' + \ + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ + " org.apache.phoenix.pherf.Pherf " + args + +exitcode = subprocess.call(java_cmd, shell=True) +sys.exit(exitcode) http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/bin/pherf-standalone.py ---------------------------------------------------------------------- diff --git a/bin/pherf-standalone.py b/bin/pherf-standalone.py new file mode 100755 index 0000000..bac2337 --- /dev/null +++ b/bin/pherf-standalone.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +import os +import subprocess +import sys +import phoenix_utils + +phoenix_utils.setPath() + +args = phoenix_utils.shell_quote(sys.argv[1:]) + +# HBase configuration folder path (where hbase-site.xml reside) for +# HBase/Phoenix client side property override +hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir) + +java_home = os.getenv('JAVA_HOME') + +# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR +hbase_env_path = None +hbase_env_cmd = None +if os.name == 'posix': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh') + hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path] +elif os.name == 'nt': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd') + hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path] +if not hbase_env_path or not hbase_env_cmd: + print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name + sys.exit(-1) + +hbase_env = {} +if os.path.isfile(hbase_env_path): + p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE) + for x in p.stdout: + (k, _, v) = x.partition('=') + hbase_env[k.strip()] = v.strip() + +if hbase_env.has_key('JAVA_HOME'): + java_home = hbase_env['JAVA_HOME'] + +if java_home: + java = os.path.join(java_home, 'bin', 'java') +else: + java = 'java' + +java_cmd = java +' -Xms512m -Xmx3072m -cp "' + phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.phoenix_pherf_jar + \ + '" -Dlog4j.configuration=file:' + \ + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ + " org.apache.phoenix.pherf.Pherf " + args + +exitcode = subprocess.call(java_cmd, shell=True) +sys.exit(exitcode) http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/bin/phoenix_utils.py ---------------------------------------------------------------------- diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py index cb48194..068d097 100755 --- a/bin/phoenix_utils.py +++ b/bin/phoenix_utils.py @@ -70,6 +70,7 @@ def setPath(): PHOENIX_QUERYSERVER_JAR_PATTERN = "phoenix-server-*-runnable.jar" PHOENIX_TRACESERVER_JAR_PATTERN = "phoenix-tracing-webapp-*-runnable.jar" PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar" + PHOENIX_PHERF_JAR_PATTERN = "phoenix-pherf-*-minimal*.jar" # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH global phoenix_class_path @@ -93,6 +94,12 @@ def setPath(): global current_dir current_dir = os.path.dirname(os.path.abspath(__file__)) + global pherf_conf_path + pherf_conf_path = os.path.join(current_dir, "config") + pherf_properties_file = find("pherf.properties", pherf_conf_path) + if pherf_properties_file == "": + pherf_conf_path = os.path.join(current_dir, "..", "phoenix-pherf", "config") + global phoenix_jar_path phoenix_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*") @@ -148,6 +155,12 @@ def setPath(): if phoenix_traceserver_jar == "": phoenix_traceserver_jar = findFileInPathWithoutRecursion(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, "..")) + global phoenix_pherf_jar + phoenix_pherf_jar = find(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-pherf", "target", "*")) + if phoenix_pherf_jar == "": + phoenix_pherf_jar = findFileInPathWithoutRecursion(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, "..", "lib")) + if phoenix_pherf_jar == "": + phoenix_pherf_jar = findFileInPathWithoutRecursion(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, "..")) global phoenix_thin_client_jar phoenix_thin_client_jar = find(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server-client", "target", "*")) http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/dev/make_rc.sh ---------------------------------------------------------------------- diff --git a/dev/make_rc.sh b/dev/make_rc.sh index cee493e..288bfdf 100755 --- a/dev/make_rc.sh +++ b/dev/make_rc.sh @@ -37,6 +37,7 @@ REL_SRC=phoenix-$PHOENIX-src DIR_REL_SRC_TAR_PATH=$DIR_REL_ROOT/src DIR_REL_BIN_TAR_PATH=$DIR_REL_ROOT/bin DIR_BIN=$DIR_REL_BIN_PATH/bin +DIR_PHERF_CONF=phoenix-pherf/config DIR_EXAMPLES=$DIR_REL_BIN_PATH/examples DIR_DOCS=dev/release_files @@ -75,12 +76,9 @@ rm -rf $(find . -type d -name archive-tmp); phx_jars=$(find -iname phoenix-*.jar) cp $phx_jars $DIR_REL_BIN_PATH; -# Exclude uber jars as it exceeds max upload limit for Apache SVN until PHOENIX-2248 is fixed -rm $DIR_REL_BIN_PATH/*webapp*runnable* -rm $DIR_REL_BIN_PATH/*pherf*dependencies* - # Copy bin cp bin/* $DIR_BIN; +cp -R $DIR_PHERF_CONF $DIR_BIN; # Copy release docs http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/README.md ---------------------------------------------------------------------- diff --git a/phoenix-pherf/README.md b/phoenix-pherf/README.md index e3e4213..c813f74 100644 --- a/phoenix-pherf/README.md +++ b/phoenix-pherf/README.md @@ -100,6 +100,6 @@ Default quorum is localhost. If you want to override set the system variable. Run unit tests: `mvn test -DZK_QUORUM=localhost`<br /> Run a specific method: `mvn -Dtest=ClassName#methodName test` <br /> -To test on a real cluster: `./pherf.sh -drop all -l -q -z localhost -schemaFile .*user_defined_schema.sql -scenarioFile .*user_defined_scenario.xml` +To test: `./pherf-standalone.py -drop all -l -q -z localhost -schemaFile .*user_defined_schema.sql -scenarioFile .*user_defined_scenario.xml` More to come... http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/cluster/pherf.sh ---------------------------------------------------------------------- diff --git a/phoenix-pherf/cluster/pherf.sh b/phoenix-pherf/cluster/pherf.sh deleted file mode 100755 index 2e0913e..0000000 --- a/phoenix-pherf/cluster/pherf.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source config/env.sh -HBASE_CLASSPATH=`$HBASE_PATH/bin/hbase classpath` - - -PHERF_HOME=$(cd "`dirname $0`" && pwd) -CLASSPATH=${HBASE_CLASSPATH} -CLASSPATH=${PHERF_HOME}/config:${CLASSPATH} - -for f in $PHERF_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -CMD="time ${JAVA_HOME}/bin/java ${REMOTE_DEBUG} -Dapp.home=${PHERF_HOME} ${ENV_PROPS} -Xms512m -Xmx3072m -cp ${CLASSPATH} org.apache.phoenix.pherf.Pherf ${@}" - -eval $CMD http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/config/pherf.properties ---------------------------------------------------------------------- diff --git a/phoenix-pherf/config/pherf.properties b/phoenix-pherf/config/pherf.properties index de5c167..bfce8bc 100644 --- a/phoenix-pherf/config/pherf.properties +++ b/phoenix-pherf/config/pherf.properties @@ -36,4 +36,4 @@ pherf.default.results.dir=RESULTS pherf.default.summary.file=RESULTS/summary.html # Threshold for comparator to fail. ex. 0.5 equates to 50% -pherf.default.comparison.threshold=0.45 \ No newline at end of file +pherf.default.comparison.threshold=0.45 http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml index 6545c1e..e3d375c 100644 --- a/phoenix-pherf/pom.xml +++ b/phoenix-pherf/pom.xml @@ -108,6 +108,10 @@ </resource> <resource> <directory>config</directory> + <excludes> + <exclude>**/*.sql</exclude> + <exclude>**/*.xml</exclude> + </excludes> </resource> </resources> <testResources> @@ -177,11 +181,6 @@ <descriptors> <!-- Produces minimal Pherf jar --> <descriptor>src/main/assembly/minimal.xml</descriptor> - <!-- Produces standalone zip that bundles all required dependencies --> - <descriptor>src/main/assembly/standalone.xml</descriptor> - <!-- Produces cluster zip with minimal Pherf jar. Setting HBase classpath - in env.sh is required for this configuration --> - <descriptor>src/main/assembly/cluster.xml</descriptor> </descriptors> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java index 3d504fd..b6e509d 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java @@ -37,7 +37,7 @@ public class PherfConstants { public static final String RESOURCE_SCENARIO = "/scenario"; public static final String SCENARIO_ROOT_PATTERN = - ".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*"; + ".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*"; public static final String SCHEMA_ROOT_PATTERN = ".*"; public static final String PHERF_PROPERTIES = "pherf.properties"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java index 06a046d..2b1689a 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java @@ -130,7 +130,7 @@ public class RulesApplier { prefix = column.getPrefix(); } - if (prefix.length() >= length) { + if ((prefix.length() >= length) && (length > 0)) { logger.warn("You are attempting to generate data with a prefix (" + prefix + ") " + "That is longer than expected overall field length (" + length + "). " + "This will certainly lead to unexpected data values."); http://git-wip-us.apache.org/repos/asf/phoenix/blob/6a37307c/phoenix-pherf/standalone/pherf.sh ---------------------------------------------------------------------- diff --git a/phoenix-pherf/standalone/pherf.sh b/phoenix-pherf/standalone/pherf.sh deleted file mode 100755 index 2b91d2c..0000000 --- a/phoenix-pherf/standalone/pherf.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source config/env.sh -PHERF_HOME=$(cd "`dirname $0`" && pwd) -CLASSPATH=${PHERF_HOME}/config:${CLASSPATH} - -for f in $PHERF_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -CMD="time ${JAVA_HOME}/bin/java ${REMOTE_DEBUG} -Dapp.home=${PHERF_HOME} ${ENV_PROPS} -Xms512m -Xmx3072m -cp ${CLASSPATH} org.apache.phoenix.pherf.Pherf ${@}" -eval $CMD \ No newline at end of file
