Repository: ambari Updated Branches: refs/heads/branch-2.5 ac32c2885 -> db4aef04e
AMBARI-18307. Add sqlline capability to AMS collector to help debugging. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db4aef04 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db4aef04 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db4aef04 Branch: refs/heads/branch-2.5 Commit: db4aef04e99697d4681e592294889c925bbd64ce Parents: ac32c28 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Tue Oct 18 15:56:11 2016 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Tue Oct 18 15:56:30 2016 -0700 ---------------------------------------------------------------------- ambari-metrics/ambari-metrics-assembly/pom.xml | 13 ++ .../conf/unix/sqlline/log4j.properties | 76 ++++++++ .../conf/unix/sqlline/phoenix_utils.py | 193 +++++++++++++++++++ .../conf/unix/sqlline/sqlline.py | 105 ++++++++++ .../ambari-metrics-timelineservice/pom.xml | 13 ++ 5 files changed, 400 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index f6cb361..a55ae57 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -309,6 +309,19 @@ </sources> </mapping> <mapping> + <directory>/usr/lib/ambari-metrics-collector/bin</directory> + <filemode>755</filemode> + <username>root</username> + <groupname>root</groupname> + <directoryIncluded>false</directoryIncluded> + <sources> + <source> + <location>${collector.dir}/conf/unix/sqlline</location> + <filter>false</filter> + </source> + </sources> + </mapping> + <mapping> <directory>/etc/ambari-metrics-collector/conf</directory> <configuration>true</configuration> <sources> http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties new file mode 100644 index 0000000..3f7760a --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +# Define some default values that can be overridden by system properties +psql.root.logger=WARN,console +psql.log.dir=. +psql.log.file=psql.log +hadoop.log.dir=. + +# Define the root logger to the system property "plsql.root.logger". +log4j.rootLogger=${psql.root.logger} + +# Logging Threshold to INFO for query-server. root logger still at WARN for sqlline clients. +log4j.threshold=INFO + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${psql.log.dir}/${psql.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Custom Logging levels +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper=ERROR +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=ERROR +log4j.logger.org.apache.hadoop.hbase.HBaseConfiguration=ERROR + +# query server packages +log4j.logger.org.apache.calcite.avatica=INFO +log4j.logger.org.apache.phoenix.queryserver.server=INFO +log4j.logger.org.eclipse.jetty.server=INFO http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py new file mode 100644 index 0000000..6d9b791 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +import os +import fnmatch +import subprocess + +def find(pattern, classPaths): + paths = classPaths.split(os.pathsep) + + # for each class path + for path in paths: + # remove * if it's at the end of path + if ((path is not None) and (len(path) > 0) and (path[-1] == '*')) : + path = path[:-1] + + for root, dirs, files in os.walk(path): + # sort the file names so *-client always precedes *-thin-client + files.sort() + for name in files: + if fnmatch.fnmatch(name, pattern): + return os.path.join(root, name) + + return "" + +def findFileInPathWithoutRecursion(pattern, path): + if not os.path.exists(path): + return "" + files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] + # sort the file names so *-client always precedes *-thin-client + files.sort() + for name in files: + if fnmatch.fnmatch(name, pattern): + return os.path.join(path, name) + + return "" + +def which(file): + for path in os.environ["PATH"].split(os.pathsep): + if os.path.exists(os.path.join(path, file)): + return os.path.join(path, file) + return None + +def findClasspath(file): + aPath = which(file) + command = "%s%s" %(aPath, ' classpath') + return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read() + +def setPath(): + PHOENIX_CLIENT_JAR_PATTERN = "phoenix-*-client.jar" + PHOENIX_THIN_CLIENT_JAR_PATTERN = "phoenix-*-thin-client.jar" + PHOENIX_QUERYSERVER_JAR_PATTERN = "phoenix-server-*-runnable.jar" + PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar" + + # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH + global phoenix_class_path + phoenix_class_path = os.getenv('PHOENIX_LIB_DIR','') + if phoenix_class_path == "": + phoenix_class_path = os.getenv('PHOENIX_CLASS_PATH','') + + global hbase_conf_dir + # if HBASE_CONF_DIR set explicitly, use that + hbase_conf_dir = os.getenv('HBASE_CONF_DIR', os.getenv('HBASE_CONF_PATH')) + if not hbase_conf_dir: + # else fall back to HBASE_HOME + if os.getenv('HBASE_HOME'): + hbase_conf_dir = os.path.join(os.getenv('HBASE_HOME'), "conf") + elif os.name == 'posix': + # default to the bigtop configuration dir + hbase_conf_dir = '/etc/hbase/conf' + else: + # Try to provide something valid + hbase_conf_dir = '.' + global hbase_conf_path # keep conf_path around for backward compatibility + hbase_conf_path = hbase_conf_dir + + global current_dir + current_dir = os.path.dirname(os.path.abspath(__file__)) + + global phoenix_jar_path + phoenix_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*") + + global phoenix_client_jar + phoenix_client_jar = find("phoenix-*-client.jar", phoenix_jar_path) + if phoenix_client_jar == "": + phoenix_client_jar = findFileInPathWithoutRecursion(PHOENIX_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..")) + if phoenix_client_jar == "": + phoenix_client_jar = find(PHOENIX_CLIENT_JAR_PATTERN, phoenix_class_path) + + global phoenix_test_jar_path + phoenix_test_jar_path = os.path.join(current_dir, "..", "phoenix-core", "target","*") + + global hadoop_conf + hadoop_conf = os.getenv('HADOOP_CONF_DIR', None) + if not hadoop_conf: + if os.name == 'posix': + # Try to provide a sane configuration directory for Hadoop if not otherwise provided. + # If there's no jaas file specified by the caller, this is necessary when Kerberos is enabled. + hadoop_conf = '/etc/hadoop/conf' + else: + # Try to provide something valid.. + hadoop_conf = '.' + + global hadoop_classpath + if (os.name != 'nt'): + hadoop_classpath = findClasspath('hadoop') + else: + hadoop_classpath = os.getenv('HADOOP_CLASSPATH', '') + + global hadoop_common_jar_path + hadoop_common_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*") + + global hadoop_common_jar + hadoop_common_jar = find("hadoop-common*.jar", hadoop_common_jar_path) + + global hadoop_hdfs_jar_path + hadoop_hdfs_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*") + + global hadoop_hdfs_jar + hadoop_hdfs_jar = find("hadoop-hdfs*.jar", hadoop_hdfs_jar_path) + + global testjar + testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_test_jar_path) + if testjar == "": + testjar = findFileInPathWithoutRecursion(PHOENIX_TESTS_JAR_PATTERN, os.path.join(current_dir, "..")) + if testjar == "": + testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_class_path) + + global phoenix_queryserver_jar + phoenix_queryserver_jar = find(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server", "target", "*")) + if phoenix_queryserver_jar == "": + phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, "..", "lib")) + if phoenix_queryserver_jar == "": + phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, "..")) + + + global phoenix_thin_client_jar + phoenix_thin_client_jar = find(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server-client", "target", "*")) + if phoenix_thin_client_jar == "": + phoenix_thin_client_jar = findFileInPathWithoutRecursion(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..")) + + return "" + +def shell_quote(args): + """ + Return the platform specific shell quoted string. Handles Windows and *nix platforms. + + :param args: array of shell arguments + :return: shell quoted string + """ + if os.name == 'nt': + import subprocess + return subprocess.list2cmdline(args) + else: + # pipes module isn't available on Windows + import pipes + return " ".join([pipes.quote(v) for v in args]) + +if __name__ == "__main__": + setPath() + print "phoenix_class_path:", phoenix_class_path + print "hbase_conf_dir:", hbase_conf_dir + print "hbase_conf_path:", hbase_conf_path + print "current_dir:", current_dir + print "phoenix_jar_path:", phoenix_jar_path + print "phoenix_client_jar:", phoenix_client_jar + print "phoenix_test_jar_path:", phoenix_test_jar_path + print "hadoop_common_jar_path:", hadoop_common_jar_path + print "hadoop_common_jar:", hadoop_common_jar + print "hadoop_hdfs_jar_path:", hadoop_hdfs_jar_path + print "hadoop_hdfs_jar:", hadoop_hdfs_jar + print "testjar:", testjar + print "phoenix_queryserver_jar:", phoenix_queryserver_jar + print "phoenix_thin_client_jar:", phoenix_thin_client_jar + print "hadoop_classpath:", hadoop_classpath http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py new file mode 100644 index 0000000..852f264 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +import os +import subprocess +import sys +import phoenix_utils +import atexit +import fnmatch + +# Set JAVA_HOME or JDK_HOME before running sqlline.py +# Example: ./sqlline.py localhost:61181:/ams-hbase-unsecure + +global childProc +childProc = None +def kill_child(): + if childProc is not None: + childProc.terminate() + childProc.kill() + if os.name != 'nt': + os.system("reset") +atexit.register(kill_child) + +def find_java(): + try: + java = os.environ.get('JAVA_HOME') + if not java: + java = os.environ.get('JDK_HOME') + except: + java = None + if java and os.path.exists(java): + if java.endswith('bin') or java.endswith('bin/'): + return os.path.join(java, 'java') + if java.find('/bin/java') == -1: + return os.path.join(java, 'bin', 'java') + else: + return 'java' + +#phoenix_utils.setPath() + +if len(sys.argv) < 2: + print "Zookeeper not specified. \nUsage: sqlline.py <zookeeper> \ +<optional_sql_file> \nExample: \n 1. sqlline.py localhost:2181:/hbase \n 2. sqlline.py \ +localhost:2181:/hbase ../examples/stock_symbol.sql" + sys.exit() + +sqlfile = "" + +if len(sys.argv) > 2: + sqlfile = "--run=" + phoenix_utils.shell_quote([sys.argv[2]]) + +colorSetting = "true" +# disable color setting for windows OS +if os.name == 'nt': + colorSetting = "false" + +# Set custom path vars +hbase_conf_dir = os.environ.get('HBASE_CONF_DIR', '/etc/ams-hbase/conf') +hadoop_conf = os.environ.get('HADOOP_CONF_DIR', '/etc/ams-hbase/conf') +hadoop_classpath = os.environ.get('HADOOP_CLASSPATH', '/usr/lib/ams-hbase/lib') + +PATH_PREFIX = '/usr/lib/ambari-metrics-collector/' +if not os.path.exists(PATH_PREFIX): + print 'Fatal Error: ' + PATH_PREFIX + ' not a valid path.' + sys.exit(1) + +current_dir = os.path.dirname(os.path.abspath(__file__)) +phoenix_client_path = PATH_PREFIX + '*' + +java_cmd = find_java() + ' -cp "' + phoenix_client_path + \ + os.pathsep + hadoop_conf + '" -Dlog4j.configuration=file:' + \ + os.path.join(current_dir, "log4j.properties") + \ + " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \ +-u jdbc:phoenix:" + phoenix_utils.shell_quote([sys.argv[1]]) + \ + " -n none -p none --color=" + colorSetting + " --fastConnect=false --verbose=true \ +--isolation=TRANSACTION_READ_COMMITTED " + sqlfile + +print 'java command: %s' % str(java_cmd) + +childProc = subprocess.Popen(java_cmd, shell=True) +#Wait for child process exit +(output, error) = childProc.communicate() +returncode = childProc.returncode +childProc = None +# Propagate Java return code to this script +if returncode is not None: + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 1144573..fdbfdcb 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -182,6 +182,19 @@ </sources> </mapping> <mapping> + <directory>/usr/lib/ambari-metrics-collector/bin</directory> + <filemode>755</filemode> + <username>root</username> + <groupname>root</groupname> + <directoryIncluded>false</directoryIncluded> + <sources> + <source> + <location>conf/unix/sqlline</location> + <filter>false</filter> + </source> + </sources> + </mapping> + <mapping> <directory>/etc/ambari-metrics-collector/conf</directory> <configuration>true</configuration> <sources>