http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/eagle-singlenode.json ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/eagle-singlenode.json b/eagle-external/eagle-docker/resource/eagle-singlenode.json deleted file mode 100644 index a693c7d..0000000 --- a/eagle-external/eagle-docker/resource/eagle-singlenode.json +++ /dev/null @@ -1,122 +0,0 @@ -{ -"configurations": [ - { - "hdfs-site": { - "dfs.permissions.enabled": "false" - }, - "hive-site": { - "javax.jdo.option.ConnectionUserName": "hive", - "javax.jdo.option.ConnectionPassword": "hive" - } - }, - { - "hadoop-env": { - "properties" : { - "content" : "\r\n# Set Hadoop-specific environment variables here.\r\n\r\n# The only required environment variable is JAVA_HOME. All others are\r\n# optional. When running a distributed configuration it is best to\r\n# set JAVA_HOME in this file, so that it is correctly defined on\r\n# remote nodes.\r\n\r\n# The java implementation to use. Required.\r\nexport JAVA_HOME={{java_home}}\r\nexport HADOOP_HOME_WARN_SUPPRESS=1\r\n\r\n# Hadoop home directory\r\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\r\n\r\n# Hadoop Configuration Directory\r\n\r\n{# this is different for HDP1 #}\r\n# Path to jsvc required by secure HDP 2.0 datanode\r\nexport JSVC_HOME={{jsvc_path}}\r\n\r\n\r\n# The maximum amount of heap to use, in MB. Default is 1000.\r\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\r\n\r\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\r\n\r\n# Extra Java runtime options. Empty by default.\r\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=tr ue ${HADOOP_OPTS}\"\r\n\r\n# Command specific options appended to HADOOP_OPTS when specified\r\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT ${HADOOP_NAMENODE_OPTS}\"\r\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d %H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\r\n\r\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\r\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\r\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCE R_OPTS}\"\r\n\r\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\r\n\r\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\r\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\r\n\r\n# On secure datanodes, user to run the datanode as after dropping privileges\r\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\r\n\r\n# Extra ssh options. Empty by default.\r\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\r\n\r\n# Where log files are stored. $HADOOP_HOME/logs by default.\r\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\r\n\r\n# History server logs\r\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\r\n\r\n# Where log files are stored in the secure data environment.\r\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\r\n# e xport HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\r\n\r\n# host:path where hadoop code should be rsync'd from. Unset by default.\r\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\r\n\r\n# Seconds to sleep between slave commands. Unset by default. This\r\n# can be useful in large clusters, where, e.g., slave rsyncs can\r\n# otherwise arrive faster than the master can service them.\r\n# export HADOOP_SLAVE_SLEEP=0.1\r\n\r\n# The directory where pid files are stored. /tmp by default.\r\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\r\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# History server pid\r\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\r\n\r\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\r\n\r\n# A string representing this instance of hadoop. $USER by default.\r\nexport HADOOP_IDENT_STRING=$USER\r\n\r\n# The scheduling priority for daemon processes. See 'm an nice'.\r\n\r\n# export HADOOP_NICENESS=10\r\n\r\n# Use libraries from standard classpath\r\nJAVA_JDBC_LIBS=\"\"\r\n#Add libraries required by mysql connector\r\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\r\ndo\r\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by oracle connector\r\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\r\ndo\r\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by nodemanager\r\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\r\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}:/usr/hdp/current/eagle/lib/log4jkafka/lib/*\r\n\r\n# added to the HADOOP_CLASSPATH\r\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\r\n if [ -d \"/etc/tez/conf/\" ]; then\r\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\r\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez -client/lib/*:/etc/tez/conf/\r\n fi\r\nfi\r\n\r\n# Setting path to hdfs command line\r\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\r\n\r\n# Mostly required for hadoop 2.0\r\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\r\n\r\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\r\n" - } - } - }, - { - "hdfs-log4j": { - "properties": { - "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\ n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.eagle.incubator.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT .Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\ nlog4j.appender.RFA.MaxBackupIndex=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter =org.apache.hadoop.log.metrics.EventCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n" - } - } - } - ], - "host_groups": [ - { - "name": "master", - "components": [ - { - "name": "APP_TIMELINE_SERVER" - }, - { - "name": "HISTORYSERVER" - }, - { - "name": "HBASE_REGIONSERVER" - }, - { - "name": "WEBHCAT_SERVER" - }, - { - "name": "HCAT" - }, - { - "name": "HBASE_CLIENT" - }, - { - "name": "NAMENODE" - }, - { - "name": "AMBARI_SERVER" - }, - { - "name": "HDFS_CLIENT" - }, - { - "name": "HIVE_CLIENT" - }, - { - "name": "NODEMANAGER" - }, - { - "name": "DATANODE" - }, - { - "name": "RESOURCEMANAGER" - }, - { - "name": "ZOOKEEPER_SERVER" - }, - { - "name": "ZOOKEEPER_CLIENT" - }, - { - "name": "HBASE_MASTER" - }, - { - "name": "HIVE_SERVER" - }, - { - "name": "SECONDARY_NAMENODE" - }, - { - "name": "HIVE_METASTORE" - }, - { - "name": "YARN_CLIENT" - }, - { - "name": "MAPREDUCE2_CLIENT" - }, - { - "name": "MYSQL_SERVER" - }, - { "name": "GANGLIA_SERVER"}, - - { "name": "GANGLIA_MONITOR"}, - - { "name": "KAFKA_BROKER"}, - { - "name" : "DRPC_SERVER" - }, - { - "name" : "NIMBUS" - }, - { - "name" : "STORM_UI_SERVER" - }, - { "name" : "SUPERVISOR"} - ], - - "cardinality": "1" - } - ], - "Blueprints": { - "blueprint_name": "hdp-singlenode-eagle", - "stack_name": "HDP", - "stack_version": "2.2" - } -}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/install-cluster.sh ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/install-cluster.sh b/eagle-external/eagle-docker/resource/install-cluster.sh deleted file mode 100755 index 4acd384..0000000 --- a/eagle-external/eagle-docker/resource/install-cluster.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -export PATH=/usr/jdk64/jdk1.7.0_67/bin:$PATH - - -./ambari-shell.sh << EOF -blueprint add --file /tmp/eagle-singlenode.json -blueprint add --file /tmp/eagle-multinode.json -cluster build --blueprint $BLUEPRINT -cluster autoAssign -cluster create --exitOnFinish true -EOF - -clear - -SERF_RPC_ADDR=${AMBARISERVER_PORT_7373_TCP##*/} -serf event --rpc-addr=$SERF_RPC_ADDR eagle - -./wait-for-eagle.sh - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/serf/etc/ambari.json ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/serf/etc/ambari.json b/eagle-external/eagle-docker/resource/serf/etc/ambari.json deleted file mode 100644 index 4409e84..0000000 --- a/eagle-external/eagle-docker/resource/serf/etc/ambari.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "event_handlers": [ - "member-join=/usr/local/serf/handlers/ambari-bootstrap", - "user:eagle=/usr/local/serf/handlers/eagle" - ], - "tags" : { - "ambari-agent": "true" - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/serf/handlers/eagle ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/serf/handlers/eagle b/eagle-external/eagle-docker/resource/serf/handlers/eagle deleted file mode 100755 index 7f7df4b..0000000 --- a/eagle-external/eagle-docker/resource/serf/handlers/eagle +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install Eagle on the Ambari server instance -if [[ "$SERF_TAG_AMBARI_SERVER" == "true" ]] ;then - echo run eagle install script - nohup /usr/hdp/current/eagle/deploy.sh > /var/log/eagle-deploy.log -fi http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/wait-for-eagle.sh ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/wait-for-eagle.sh b/eagle-external/eagle-docker/resource/wait-for-eagle.sh deleted file mode 100755 index 3b6ab11..0000000 --- a/eagle-external/eagle-docker/resource/wait-for-eagle.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -: ${EAGLE_HOST:=$AMBARISERVER_PORT_9099_TCP_ADDR} -: ${SLEEP:=2} -: ${DEBUG:=1} - -: ${EAGLE_HOST:? eagle server address is mandatory, fallback is a linked containers exposed 9099} - -debug() { - [ $DEBUG -gt 0 ] && echo [DEBUG] "$@" 1>&2 -} - -get-server-state() { - curl -s -o /dev/null -w "%{http_code}" $AMBARISERVER_PORT_9099_TCP_ADDR:9099/eagle-service/index.html -} - -debug waits for eagle to start on: $EAGLE_HOST -while ! get-server-state | grep 200 &>/dev/null ; do - [ $DEBUG -gt 0 ] && echo -n . - sleep $SLEEP -done -[ $DEBUG -gt 0 ] && echo -debug eagle web started: $EAGLE_HOST:9099/eagle-service http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/pom.xml b/eagle-external/eagle-log4jkafka/pom.xml deleted file mode 100644 index 2c4871b..0000000 --- a/eagle-external/eagle-log4jkafka/pom.xml +++ /dev/null @@ -1,116 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>eagle-external-parent</artifactId> - <groupId>eagle</groupId> - <version>0.3.0</version> - <relativePath>../pom.xml</relativePath> - </parent> - <packaging>jar</packaging> - <modelVersion>4.0.0</modelVersion> - <artifactId>eagle-log4jkafka</artifactId> - - <dependencies> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - <!--<dependency>--> - <!--<groupId>org.slf4j</groupId>--> - <!--<artifactId>slf4j-log4j12</artifactId>--> - <!--</dependency>--> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_${scala.version}</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.version}</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <sourceDirectory>src/main/scala</sourceDirectory> - <plugins> - <plugin> - <groupId>org.scala-tools</groupId> - <artifactId>maven-scala-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>compile</goal> - <goal>testCompile</goal> - </goals> - <configuration> - <args> - <arg>-make:transitive</arg> - <arg>-dependencyfile</arg> - <arg>${project.build.directory}/.scala_dependencies</arg> - </args> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <useFile>false</useFile> - <disableXmlReport>true</disableXmlReport> - <includes> - <include>**/*Test.*</include> - <include>**/Test*.*</include> - <include>**/*Suite.*</include> - </includes> - </configuration> - </plugin> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <descriptor>src/assembly/build.xml</descriptor> - <finalName>eagle-log4jkafka</finalName> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - <configuration> - <tarLongFileMode>posix</tarLongFileMode> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/assembly/build.xml ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/assembly/build.xml b/eagle-external/eagle-log4jkafka/src/assembly/build.xml deleted file mode 100644 index 54030eb..0000000 --- a/eagle-external/eagle-log4jkafka/src/assembly/build.xml +++ /dev/null @@ -1,61 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> - <id>build</id> - <formats> - <format>dir</format> - </formats> - <includeBaseDirectory>false</includeBaseDirectory> - <dependencySets> - <dependencySet> - <outputDirectory>lib/</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <unpack>false</unpack> - <scope>runtime</scope> - <includes> - <include>org.apache.kafka:kafka_2.10</include> - <include>org.apache.kafka:kafka-clients</include> - <include>com.yammer.metrics:metrics-core</include> - <include>org.scala-lang:scala-library</include> - </includes> - </dependencySet> - </dependencySets> - <fileSets> - <fileSet> - <directory>${project.basedir}/target</directory> - <outputDirectory>lib/</outputDirectory> - <includes> - <include>*.jar</include> - </includes> - <excludes> - <exclude>*-tests.jar</exclude> - <exclude>*-sources.jar</exclude> - </excludes> - </fileSet> - <fileSet> - <directory>${project.basedir}/target/conf</directory> - <outputDirectory>conf/</outputDirectory> - <includes> - <include>*</include> - </includes> - </fileSet> - </fileSets> -</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties b/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties deleted file mode 100644 index d67ea03..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Name configure audit appender through "hdfs.audit.logger" -#hdfs.audit.logger=INFO,console, KAFKA_HDFS_AUDIT,DRFAAUDIT -#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} -#log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false -# - -log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender -log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log -log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667 -log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer -log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout -log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async -log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1 -log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala deleted file mode 100644 index b3519cc..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.log4j.kafka - -import java.util.{Date, Properties} - -import kafka.producer.async.MissingConfigException -import kafka.utils.{Logging, Utils} -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} -import org.apache.log4j.AppenderSkeleton -import org.apache.log4j.helpers.LogLog -import org.apache.log4j.spi.LoggingEvent - -class JKafkaLog4jAppender extends AppenderSkeleton with Logging { - var topic: String = null - var brokerList: String = null - var compressionType: String = null - var retries: Int = 0 - var requiredNumAcks: Int = Int.MaxValue - var syncSend: Boolean = false - - var keyClass: String = null - var keyer: Keyer = null - - var topicClass: String = null - var topicPicker: TopicPicker = null - - private var producer: KafkaProducer[Array[Byte],Array[Byte]] = null - - def getTopic: String = topic - def setTopic(topic: String) { this.topic = topic } - - def getBrokerList: String = brokerList - def setBrokerList(brokerList: String) { this.brokerList = brokerList } - - def getCompressionType: String = compressionType - def setCompressionType(compressionType: String) { this.compressionType = compressionType } - - def getRequiredNumAcks: Int = requiredNumAcks - def setRequiredNumAcks(requiredNumAcks: Int) { this.requiredNumAcks = requiredNumAcks } - - def getSyncSend: Boolean = syncSend - def setSyncSend(syncSend: Boolean) { this.syncSend = syncSend } - - def getRetries: Int = retries - def setRetries(retries: Int) { this.retries = retries } - - def getKeyClass: String = keyClass - def setKeyClass(keyClass: String) { this.keyClass = keyClass } - - def getTopicClass: String = topicClass - def setTopicClass(topicClass: String) { this.topicClass = topicClass } - - var keyPattern: String = null - - def getKeyPattern:String = keyPattern - - def setKeyPattern(keyPattern: String) { this.keyPattern = keyPattern } - - override def activateOptions() { - // check for config parameter validity - val props = new Properties() - if(brokerList != null) - props.put(org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList) - if(props.isEmpty) - throw new MissingConfigException("The bootstrap servers property should be specified") - - // if(topic == null) - if(topic == null && topicClass == null) - throw new MissingConfigException("topic must be specified by the Kafka log4j appender") - - props.put("topic",topic) - - if(keyPattern != null) props.put("keyPattern", keyPattern) - - if(compressionType != null) props.put(org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType) - if(requiredNumAcks != Int.MaxValue) props.put(org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG, requiredNumAcks.toString) - if(retries > 0) props.put(org.apache.kafka.clients.producer.ProducerConfig.RETRIES_CONFIG, retries.toString) - props.put(org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer") - props.put(org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer") - producer = new KafkaProducer[Array[Byte],Array[Byte]](props) - - if(keyClass != null){ - keyer = Utils.createObject[Keyer](keyClass,props) - LogLog.debug("Instantiated Key class " + keyClass) - } - - if(topicClass != null){ - topicPicker = Utils.createObject[TopicPicker](topicClass,props) - LogLog.debug("Instantiated Topic class: " + topicClass) - } - LogLog.debug("Kafka producer connected to " + brokerList) - LogLog.debug("Logging for topic: " + topic) - } - - override def append(event: LoggingEvent) { - val message = subAppend(event) - LogLog.debug("[" + new Date(event.getTimeStamp).toString + "]" + message) - // val response = producer.send(new ProducerRecord[Array[Byte],Array[Byte]](topic, message.getBytes())) - val keyBytes = if(keyer != null) keyer.getKey(message).getBytes() else null - val messageTopic = if(topicPicker != null) topicPicker.getTopic(message) else topic - val response = producer.send(new ProducerRecord[Array[Byte],Array[Byte]](messageTopic, keyBytes, message.getBytes())) - if (syncSend) response.get - } - - def subAppend(event: LoggingEvent): String = { - if(this.layout == null) - event.getRenderedMessage - else - this.layout.format(event) - } - - override def close() { - if(!this.closed) { - this.closed = true - producer.close() - } - } - - override def requiresLayout: Boolean = true -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala deleted file mode 100644 index ee09294..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka - -import java.util.{Date, Properties} - -import kafka.producer.async.MissingConfigException -import kafka.producer.{KeyedMessage, Producer, ProducerConfig} -import kafka.utils.{Logging, Utils} -import org.apache.log4j.AppenderSkeleton -import org.apache.log4j.helpers.LogLog -import org.apache.log4j.spi.LoggingEvent - -class KafkaLog4jAppender extends AppenderSkeleton with Logging { - var topic:String = null - var serializerClass:String = null - var brokerList:String = null - var producerType:String = null - var compressionCodec:String = null - var enqueueTimeout:String = null - var queueSize:String = null - var requiredNumAcks: Int = Int.MaxValue - - var keyClass: String = null - var keyer: Keyer = null - - var topicClass: String = null - var topicPicker: TopicPicker = null - - var batchSize:Int = 20 - - var keyPattern: String = null - - def getKeyPattern:String = keyPattern - - def setKeyPattern(keyPattern: String) { this.keyPattern = keyPattern } - - private var producer: Producer[String, String] = null - - def getTopic:String = topic - def setTopic(topic: String) { this.topic = topic } - - def getBrokerList:String = brokerList - def setBrokerList(brokerList: String) { this.brokerList = brokerList } - - def getSerializerClass:String = serializerClass - def setSerializerClass(serializerClass:String) { this.serializerClass = serializerClass } - - def getProducerType:String = producerType - def setProducerType(producerType:String) { this.producerType = producerType } - - def getCompressionCodec:String = compressionCodec - def setCompressionCodec(compressionCodec:String) { this.compressionCodec = compressionCodec } - - def getEnqueueTimeout:String = enqueueTimeout - def setEnqueueTimeout(enqueueTimeout:String) { this.enqueueTimeout = enqueueTimeout } - - def getQueueSize:String = queueSize - def setQueueSize(queueSize:String) { this.queueSize = queueSize } - - def getRequiredNumAcks:Int = requiredNumAcks - def setRequiredNumAcks(requiredNumAcks:Int) { this.requiredNumAcks = requiredNumAcks } - - def getKeyClass: String = keyClass - def setKeyClass(keyClass: String) { this.keyClass = keyClass } - - def getTopicClass: String = topicClass - def setTopicClass(topicClass: String) { this.topicClass = topicClass } - - def getBatchSize:Int = batchSize - def setBatchSize(batchSize:Int) { this.batchSize = batchSize }; - - override def activateOptions() { - // check for config parameter validity - val props = new Properties() - if(brokerList != null) - props.put("metadata.broker.list", brokerList) - if(props.isEmpty) - throw new MissingConfigException("The metadata.broker.list property should be specified") - if(topic == null) - throw new MissingConfigException("topic must be specified by the Kafka log4j appender") - if(serializerClass == null) { - serializerClass = "kafka.serializer.StringEncoder" - LogLog.debug("Using default encoder - kafka.serializer.StringEncoder") - } - - props.put("serializer.class", serializerClass) - props.put("batch.num.messages",batchSize.toString) - - if(keyPattern != null) props.put("keyPattern", keyPattern) - - //These have default values in ProducerConfig and AsyncProducerConfig. We don't care if they're not specified - if(producerType != null) props.put("producer.type", producerType) - if(compressionCodec != null) props.put("compression.codec", compressionCodec) - if(enqueueTimeout != null) props.put("queue.enqueue.timeout.ms", enqueueTimeout) - if(queueSize != null) props.put("queue.buffering.max.messages", queueSize) - if(requiredNumAcks != Int.MaxValue) props.put("request.required.acks", requiredNumAcks.toString) - - val config : ProducerConfig = new ProducerConfig(props) - - producer = new Producer[String, String](config) - - if(keyClass != null){ - keyer = Utils.createObject[Keyer](keyClass,props) - LogLog.debug("Instantiated Key class " + keyClass) - } - - if(topicClass != null) { - try { - topicPicker = Utils.createObject[TopicPicker](topicClass, props) - LogLog.debug("Instantiated Topic class: " + topicClass) - }catch { - case e: Exception =>{ - LogLog.error("Failed to instantiate Topic class: "+topicClass,e) - } - } - } - - LogLog.debug("Kafka producer connected to " + config.brokerList) - LogLog.debug("Logging for topic: " + topic) - } - - override def append(event: LoggingEvent) { - val message = subAppend(event) - LogLog.debug("[" + new Date(event.getTimeStamp).toString + "]" + message) - - val messageKey = if(keyer != null) keyer.getKey(message) else null - val messageTopic = if(topicPicker != null) topicPicker.getTopic(message) else topic - - val messageData = new KeyedMessage[String, String](messageTopic, messageKey, message) - producer.send(messageData) - } - - def subAppend(event: LoggingEvent): String = { - if(this.layout == null) - event.getRenderedMessage - else - this.layout.format(event) - } - - override def close() { - if(!this.closed) { - this.closed = true - producer.close() - } - } - - override def requiresLayout: Boolean = false -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala deleted file mode 100644 index b25ffbe..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka - -/** - * Keyer is used to derive a key from the message. A Key is used to determine kafka partition for the message. - * - * Implementations will be constructed via reflection and are required to have a constructor that takes a single - * VerifiableProperties instance--this allows passing configuration properties into the Keyer implementation. - */ -trait Keyer { - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return returns a key based on the message - */ - def getKey(value: String): String -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala deleted file mode 100644 index 8ef1ff9..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka - -/** - * TopicPicker is used to derive a topic from the message. - * - * Implementations will be constructed via reflection and are required to have a constructor that takes a single - * VerifiableProperties instance--this allows passing configuration properties into the TopicPicker implementation. - * - * @since 6/26/15 - */ -trait TopicPicker { - /** - * returns a topic based on the message. - */ - def getTopic(value: String): String -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala deleted file mode 100644 index f435d9f..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -import java.util.Properties -import org.apache.eagle.log4j.kafka.Keyer - -class AuditLogKeyer(props:Properties) extends Keyer{ - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return returns a key based on the message - */ - override def getKey(msg: String): String = if(msg != null) AuditLogKeyer.getKey(msg.split("\\s+")) else null -} - -object AuditLogKeyer{ - def getKey(fields: Array[String]):String = if(fields.length > 5) AuditLogUtils.parseUser(fields(5)) else null -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala deleted file mode 100644 index 8d206f6..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -object AuditLogUtils { - def parseUser(userText: String): String = { - var user:String = userText.substring(userText.indexOf("=")+1) - if(user.indexOf("@") > -1){ - user = user.substring(0,user.indexOf("@")) - } - user - } - - def parseIP(ipPort:String):String = ipPort.substring(0,ipPort.indexOf(':') + 1) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala deleted file mode 100644 index 38ff72a..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -import java.util.Properties -import org.apache.eagle.log4j.kafka.Keyer - -/** - * @since 6/26/15 - */ -class AuditSecurityLogKeyer(props:Properties) extends Keyer{ - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return returns a key based on the message - * - */ - val AUDITLOG_SOURCE:String = "FSNamesystem.audit:" - - override def getKey(msg: String): String = { - val fields: Array[String] = msg.split("\\s+") - if(fields.length > 3){ - if(fields(3).equals(AUDITLOG_SOURCE)) - AuditLogKeyer.getKey(fields) - else - SecurityLogKeyer.getKey(fields) - } else null - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala deleted file mode 100644 index 25d781b..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -import org.apache.log4j.spi.{Filter, LoggingEvent} - -/** - * @since 6/26/15 - */ -class FailedAuthenticationFilter extends Filter{ - - override def decide(event: LoggingEvent): Int = { - event.getMessage match { - case message: String => - val fields: Array[String] = message.split("\\s+") - if (fields.length > 5 && fields(3).equals(FailedAuthenticationFilter.AUTH_LOG_SOURCE) && fields(5).equals(FailedAuthenticationFilter.FAILED)) { - Filter.DENY - } - case _ => - } - Filter.NEUTRAL - } -} - -object FailedAuthenticationFilter{ - val AUTH_LOG_SOURCE: String = "SecurityLogger.org.apache.hadoop.ipc.Server:" - val FAILED: String = "failed" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala deleted file mode 100644 index 0deb152..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.log4j.kafka.hadoop - -import java.util.Properties -import java.util.regex.{Matcher, Pattern} -import org.apache.eagle.log4j.kafka.Keyer - -import scala.util.matching.Regex - -class GenericLogKeyer(props:Properties) extends Keyer { - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return returns a key based on the message - */ - override def getKey(msg: String): String = { - val keyPattern = props.getProperty("keyPattern"); - if(keyPattern == null) return null - val pattern = new Regex(keyPattern, "key"); - val result = pattern.findFirstMatchIn(msg).getOrElse(null); - if (result == null) { - return null; - } - return result.group("key"); - } -} - - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala deleted file mode 100644 index 4bf87da..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -import java.util.Properties - -import org.apache.eagle.log4j.kafka.Keyer - -/** - * @since 6/26/15 - */ -class SecurityLogKeyer(props:Properties) extends Keyer{ - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return returns a key based on the message - */ - override def getKey(msg: String): String = SecurityLogKeyer.getKey(msg.split("\\s+")) -} - -object SecurityLogKeyer{ - val FAILED:String = "failed" - val SUCCESSFUL: String = "successful" - - def getKey(fields:Array[String]):String = { - if (fields.length > 5) { - if (fields(5) == SUCCESSFUL) { - return if (fields.length > 7) AuditLogUtils.parseUser(fields(7)) else null - } - else if (fields(5) == FAILED) { - return if (fields.length > 7) AuditLogUtils.parseIP(fields(7)) else null - } - } - null - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala deleted file mode 100644 index 8a2e0ab..0000000 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log4j.kafka.hadoop - -import java.util.Properties -import org.apache.eagle.log4j.kafka.TopicPicker - -/** - * @since 6/26/15 - */ -class SecurityLogTopicPicker extends TopicPicker{ - private var failedAuthnTopic: String = null - private var securityTopic: String = null - - val FAILED: String = "failed" - val TOPIC: String = "Topic" - - def this(props:Properties){ - this() - val topic = props.getProperty(TOPIC) - val topics = if(topic!=null) topic.split(",") else Array(topic) - - if(topics != null){ - if(topics.length > 0) failedAuthnTopic = topics(0) - if(topics.length > 1) securityTopic = topics(1) - } - } - - /** - * returns a topic based on the message. - */ - override def getTopic(value: String): String = { - val fields = value.split("\\s+") - if(fields.length > 5 && fields(5).equals(FAILED)) - failedAuthnTopic - else - securityTopic - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties b/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties deleted file mode 100644 index 85895a8..0000000 --- a/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -log4j.rootLogger=DEBUG, console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n - -log4j.appender.KAFKA=org.apache.eagle.log4j.kafka.KafkaLog4jAppender -log4j.appender.KAFKA.Topic=test -log4j.appender.KAFKA.BrokerList=sandbox.hortonworks.com:6667 -log4j.appender.KAFKA.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer -log4j.appender.KAFKA.Layout=org.apache.log4j.PatternLayout -log4j.appender.KAFKA.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.KAFKA.ProducerType=async -log4j.appender.KAFKA.BatchSize=1 -log4j.appender.KAFKA.QueueSize=1 - -log4j.logger.eagle.kafka.producer.TestKafkaAppender$=console,KAFKA - -#log4j.logger.kafka.utils.VerifiableProperties=INFO,console http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala deleted file mode 100644 index 840dfe7..0000000 --- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.log4j.kafka - -import java.util.Properties -import org.apache.eagle.log4j.kafka.hadoop.GenericLogKeyer -import org.scalatest.{FlatSpec, Matchers} - -abstract class TestGenericLogKeyer extends FlatSpec with Matchers { - - val hdfsMsg = "2015-07-31 01:54:35,161 INFO FSNamesystem.audit: allowed=true ugi=root (auth:TOKEN) ip=/10.0.0.1 cmd=open src=/tmp/private dst=null perm=null" - val props = new Properties() - props.put("keyPattern", "ugi=(\\w+)[@\\s+]") - props.put("keyPattern2", "user=(\\w+),\\s+") - val test = new GenericLogKeyer(props) - var keyVal = test.getKey(hdfsMsg) - println(keyVal) - - val hbaseMsg = "2015-11-06 13:14:00,741 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)" - props.put("keyPattern", "user=(\\w+),\\s+") - keyVal = test.getKey(hbaseMsg) - println(keyVal) - - //props.put("keyPattern", "user=(\\w+),\\s+") - val props1 = new Properties() - val test1 = new GenericLogKeyer(props1) - keyVal = test1.getKey(hbaseMsg) - println(keyVal) - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala deleted file mode 100644 index 3bd9e0e..0000000 --- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.log4j.kafka - -import org.scalatest.{Matchers, FlatSpec} - -abstract class TestKafkaLog4jAppender extends FlatSpec with Matchers { - - val test = new KafkaLog4jAppender(); - test.topic = "sandbox_hdfs_audit_log" - test.brokerList = "sandbox.hortonworks.com:6667" - test.keyClass = "eagle.log4j.kafka.hadoop.AuditLogKeyer" - test.keyPattern = "user=(\\w+),\\s+" - test.producerType = "async" - test.activateOptions() - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/.gitignore ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/.gitignore b/eagle-external/hadoop_jmx_collector/.gitignore deleted file mode 100644 index adad264..0000000 --- a/eagle-external/hadoop_jmx_collector/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -config.json -*.pyc \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/README.md ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/README.md b/eagle-external/hadoop_jmx_collector/README.md deleted file mode 100644 index cd8d887..0000000 --- a/eagle-external/hadoop_jmx_collector/README.md +++ /dev/null @@ -1,75 +0,0 @@ -<!-- -{% comment %} -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to you under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -{% endcomment %} ---> - - -# Hadoop Jmx Collector - -These scripts help to collect Hadoop jmx and evently sent the metrics to stdout or Kafka. Tested with Python 2.7. - -### How to use it - - 1. Edit the configuration file (json file). For example: - - { - "env": { - "site": "sandbox" - }, - "input": { - "component": "namenode", - "port": "50070", - "https": false - }, - "filter": { - "monitoring.group.selected": ["hadoop", "java.lang"] - }, - "output": { - } - } - - 2. Run the scripts - - # for general use - python hadoop_jmx_kafka.py > 1.txt - - -### Edit `eagle-collector.conf` - -* input - - "port" defines the hadoop service port, such as 50070 => "namenode", 60010 => "hbase master". - -* filter - - "monitoring.group.selected" can filter out beans which we care about. - -* output - - if we left it empty, then the output is stdout by default. - - "output": {} - - It also supports Kafka as its output. - - "output": { - "kafka": { - "topic": "test_topic", - "brokerList": [ "sandbox.hortonworks.com:6667"] - } - } - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/config-sample.json ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/config-sample.json b/eagle-external/hadoop_jmx_collector/config-sample.json deleted file mode 100644 index 57c9aae..0000000 --- a/eagle-external/hadoop_jmx_collector/config-sample.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "env": { - "site": "sandbox", - "name_node": { - "hosts":["sandbox.hortonworks.com"], - "port":50070, - "https":false - }, - "resource_manager":{ - "hosts":["sandbox.hortonworks.com"], - "port":50030, - "https":false - } - }, - "input": { - "component": "namenode", - "port": "50070", - "https": false - }, - "filter": { - "monitoring.group.selected": ["hadoop", "java.lang"] - }, - "output": { - "kafka": { - "topic": "nn_jmx_metric_sandbox", - "brokerList": ["sandbox.hortonworks.com:6667"] - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py deleted file mode 100644 index 2a5096b..0000000 --- a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py +++ /dev/null @@ -1,128 +0,0 @@ -# !/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from metric_collector import MetricCollector, JmxReader, YarnWSReader, Runner -import logging - -class HadoopNNHAChecker(MetricCollector): - def run(self): - if not self.config["env"].has_key("name_node"): - logging.warn("Do nothing for HadoopNNHAChecker as config of env.name_node not found") - return - name_node_config = self.config["env"]["name_node"] - hosts = name_node_config["hosts"] - port = name_node_config["port"] - https = name_node_config["https"] - - total_count = len(hosts) - - self.collect({ - "component": "namenode", - "metric": "hadoop.namenode.hastate.total.count", - "value": total_count - }) - - active_count = 0 - standby_count = 0 - failed_count = 0 - - for host in hosts: - try: - bean = JmxReader(host, port, https).open().get_jmx_bean_by_name( - "Hadoop:service=NameNode,name=FSNamesystem") - logging.debug(host + " is " + bean["tag.HAState"]) - if bean["tag.HAState"] == "active": - active_count += 1 - else: - standby_count += 1 - except Exception as e: - logging.exception("failed to read jmx from " + host) - failed_count += 1 - - self.collect({ - "component": "namenode", - "metric": "hadoop.namenode.hastate.active.count", - "value": active_count - }) - - self.collect({ - "component": "namenode", - "metric": "hadoop.namenode.hastate.standby.count", - "value": standby_count - }) - - self.collect({ - "component": "namenode", - "metric": "hadoop.namenode.hastate.failed.count", - "value": failed_count - }) - - -class HadoopRMHAChecker(MetricCollector): - def run(self): - if not self.config["env"].has_key("resource_manager"): - logging.warn("Do nothing for HadoopRMHAChecker as config of env.resource_manager not found") - return - name_node_config = self.config["env"]["resource_manager"] - hosts = name_node_config["hosts"] - port = name_node_config["port"] - https = name_node_config["https"] - - total_count = len(hosts) - - self.collect({ - "component": "namenode", - "metric": "hadoop.resourcemanager.hastate.total.count", - "value": total_count - }) - - active_count = 0 - standby_count = 0 - failed_count = 0 - - for host in hosts: - try: - cluster_info = YarnWSReader(host, port, https).read_cluster_info() - if cluster_info["clusterInfo"]["haState"] == "ACTIVE": - active_count += 1 - else: - standby_count += 1 - except Exception as e: - logging.exception("Failed to read yarn ws from " + host) - failed_count += 1 - - self.collect({ - "component": "resourcemanager", - "metric": "hadoop.resourcemanager.hastate.active.count", - "value": active_count - }) - - self.collect({ - "component": "resourcemanager", - "metric": "hadoop.resourcemanager.hastate.standby.count", - "value": standby_count - }) - - self.collect({ - "component": "resourcemanager", - "metric": "hadoop.resourcemanager.hastate.failed.count", - "value": failed_count - }) - -if __name__ == '__main__': - Runner.run(HadoopNNHAChecker(), HadoopRMHAChecker()) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py b/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py deleted file mode 100644 index 8837c30..0000000 --- a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py +++ /dev/null @@ -1,79 +0,0 @@ -# !/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from metric_collector import JmxMetricCollector,JmxMetricListener,Runner -import json - -class NNSafeModeMetric(JmxMetricListener): - def on_metric(self, metric): - if metric["metric"] == "hadoop.namenode.fsnamesystemstate.fsstate": - if metric["value"] == "safeMode": - metric["value"] = 1 - else: - metric["value"] = 0 - self.collector.collect(metric) - - -class NNHAMetric(JmxMetricListener): - PREFIX = "hadoop.namenode.fsnamesystem" - - def on_bean(self, bean): - if bean["name"] == "Hadoop:service=NameNode,name=FSNamesystem": - if bean[u"tag.HAState"] == "active": - self.collector.on_bean_kv(self.PREFIX, "hastate", 0) - else: - self.collector.on_bean_kv(self.PREFIX, "hastate", 1) - - -class MemortUsageMetric(JmxMetricListener): - PREFIX = "hadoop.namenode.jvm" - - def on_bean(self, bean): - if bean["name"] == "Hadoop:service=NameNode,name=JvmMetrics": - memnonheapusedusage = round(float(bean['MemNonHeapUsedM']) / float(bean['MemNonHeapMaxM']) * 100.0, 2) - self.collector.on_bean_kv(self.PREFIX, "memnonheapusedusage", memnonheapusedusage) - memnonheapcommittedusage = round(float(bean['MemNonHeapCommittedM']) / float(bean['MemNonHeapMaxM']) * 100, - 2) - self.collector.on_bean_kv(self.PREFIX, "memnonheapcommittedusage", memnonheapcommittedusage) - memheapusedusage = round(float(bean['MemHeapUsedM']) / float(bean['MemHeapMaxM']) * 100, 2) - self.collector.on_bean_kv(self.PREFIX, "memheapusedusage", memheapusedusage) - memheapcommittedusage = round(float(bean['MemHeapCommittedM']) / float(bean['MemHeapMaxM']) * 100, 2) - self.collector.on_bean_kv(self.PREFIX, "memheapcommittedusage", memheapcommittedusage) - - -class JournalTransactionInfoMetric(JmxMetricListener): - PREFIX = "hadoop.namenode.journaltransaction" - - def on_bean(self, bean): - if bean.has_key("JournalTransactionInfo"): - JournalTransactionInfo = json.loads(bean.get("JournalTransactionInfo")) - LastAppliedOrWrittenTxId = float(JournalTransactionInfo.get("LastAppliedOrWrittenTxId")) - MostRecentCheckpointTxId = float(JournalTransactionInfo.get("MostRecentCheckpointTxId")) - self.collector.on_bean_kv(self.PREFIX, "LastAppliedOrWrittenTxId", LastAppliedOrWrittenTxId) - self.collector.on_bean_kv(self.PREFIX, "MostRecentCheckpointTxId", MostRecentCheckpointTxId) - - -if __name__ == '__main__': - collector = JmxMetricCollector() - collector.register( - NNSafeModeMetric(), - NNHAMetric(), - MemortUsageMetric(), - JournalTransactionInfoMetric() - ) - Runner.run(collector) http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore b/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore deleted file mode 100644 index 260dbea..0000000 --- a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Eagle - Git Ignore Configuration -# -# See: https://github.com/github/gitignore/ - -*.egg-info -*.pyc -.tox -build -dist -MANIFEST -env -servers/*/kafka-bin -.coverage -.noseids -docs/_build http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules ---------------------------------------------------------------------- diff --git a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules b/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules deleted file mode 100644 index fe0f4fb..0000000 --- a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Eagle - Git Ignore Configuration -# -# See: https://github.com/github/gitignore/ \ No newline at end of file
