http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh deleted file mode 100644 index 327bec1..0000000 --- a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh +++ /dev/null @@ -1,50 +0,0 @@ -##!/bin/bash -# -## Licensed to the Apache Software Foundation (ASF) under one or more -## contributor license agreements. See the NOTICE file distributed with -## this work for additional information regarding copyright ownership. -## The ASF licenses this file to You under the Apache License, Version 2.0 -## (the "License"); you may not use this file except in compliance with -## the License. You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -# -source $(dirname $0)/eagle-env.sh - -###################################################################### -## Import stream metadata for HADOOP METRIC -###################################################################### -${EAGLE_HOME}/bin/hadoop-metric-init.sh - -###################################################################### -## Run topology for HADOOP METRIC -###################################################################### -### if topology exists, we should shutdown it -echo "check topology status ..." -active=$(${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology status | grep ACTIVE) -echo "topology status $active" -if [ "$active" ]; then - echo "stop topology ..." - ${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology stop -fi -echo "start Eagle Hadoop Metric Monitoring topology" -${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor --topology sandbox-hadoopjmx-topology --config ${EAGLE_HOME}/conf/sandbox-hadoopjmx-topology.conf start - -###################################################################### -## Setup minutely crontab job for HADOOP METRIC -###################################################################### -echo "set up crontab script" -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cp $DIR/../tools/hadoop_jmx_collector/config-sample.json $DIR/../tools/hadoop_jmx_collector/config.json -command="python $DIR/../tools/hadoop_jmx_collector/hadoop_jmx_kafka.py" -job="* * * * * $command >> $DIR/../logs/hadoop_metric.log" -echo "$job" -cat <(fgrep -i -v "$command" <(crontab -l)) <(echo "$job") | crontab - - -echo "$(crontab -l)"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh b/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh deleted file mode 100755 index 0d50376..0000000 --- a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with` -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/eagle-env.sh - -echo "Importing metadata for HDFS security log... " - -#### AlertStreamService: alert streams generated from data source - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream"},"desc":"alert event stream from HDFS security audit log"}]' - - -#### AlertExecutorService: what alert streams are consumed by alert executor - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hdfsSecurityLog","alertExecutorId":"hdfsSecurityLogAlertExecutor","streamName":"hdfsSecurityLogEventStream"},"desc":"alert executor for HDFS security log event stream"}]' - - -#### AlertStreamSchemaService: schema for event from alert stream - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"; -d '[{"prefix":"alertStreamSchema","tags":{"application":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"timestamp"},"attrDescription":"milliseconds of the datetime","attrType":"long","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"application":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"user"},"attrDescription":"","attrType":"string","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"application":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"allowed"},"attrDescription":"true, false or none","attrType":"bool","category":"","attrValueResolver":""}]' - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService"; -d '[{"prefix":"alertDataSource","tags":{"site":"sandbox","application":"hdfsSecurityLog"},"enabled":"true","desc":"HDFS Security"}]' - - -echo "" - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/bin/kafka-stream-monitor.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/kafka-stream-monitor.sh b/eagle-assembly/src/main/bin/kafka-stream-monitor.sh deleted file mode 100644 index bf9181b..0000000 --- a/eagle-assembly/src/main/bin/kafka-stream-monitor.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/eagle-env.sh - -if [ -e ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar ];then - export jar_name=$(ls ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar) -else - echo "ERROR: ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar not found" - exit 1 -fi - -export main_class=org.apache.eagle.datastream.storm.KafkaStreamMonitor - -export alert_stream=$1 -export alert_executor=$2 -export config_file=$3 - -if [ "$#" -lt "2" ];then - echo "ERROR: parameter required" - echo "Usage: kafka-stream-monitor.sh [alert_stream alert_executor config_file] or [alert_stream config_file]" - echo "" - exit 1 -fi -if [ "$#" == "2" ];then - config_file=$2 - alert_executor="${alert_stream}Executor" -fi - -which storm >/dev/null 2>&1 -if [ $? != 0 ];then - echo "ERROR: storm not found" - exit 1 -else - export EAGLE_CLASSPATH=$EAGLE_CLASSPATH:$jar_name:`storm classpath` -fi - -cmd="java -cp $EAGLE_CLASSPATH $main_class -D eagle.stream.name=$alert_stream -D eagle.stream.executor=$alert_executor -D config.file=$config_file -D envContextConfig.jarFile=$jar_name" - -echo "==========" -echo "Alert Stream: $alert_stream" -echo "Alert Executor: $alert_executor" -echo "Alert Config: $config_file" -echo "==========" -echo "Run Cmd: $cmd" -echo $cmd -$cmd \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/bin/pipeline-runner.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/pipeline-runner.sh b/eagle-assembly/src/main/bin/pipeline-runner.sh deleted file mode 100755 index 65bf3f3..0000000 --- a/eagle-assembly/src/main/bin/pipeline-runner.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/eagle-env.sh - -if [ -e ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar ];then - export jar_name=$(ls ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar) -else - echo "ERROR: ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar not found" - exit 1 -fi -if [ -e ${EAGLE_HOME}/conf/pipeline.conf ];then - export common_conf=$(ls ${EAGLE_HOME}/conf/pipeline.conf) -else - echo "WARN: ${EAGLE_HOME}/conf/pipeline.conf not found" -fi - -main_class=org.apache.eagle.stream.pipeline.Pipeline - -if [ "$#" == "0" ];then - echo "ERROR: parameter required" - echo "Usage: $0 [pipeline-config]" - - echo "" - exit 1 -fi - -which storm >/dev/null 2>&1 -if [ $? != 0 ];then - echo "ERROR: storm not found" - exit 1 -else - export EAGLE_CLASSPATH=$EAGLE_CLASSPATH:$jar_name:`storm classpath` -fi - -cmd="java -cp $EAGLE_CLASSPATH $main_class --conf $common_conf --pipeline $1 -D envContextConfig.jarFile=$jar_name" -echo $cmd -$cmd \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/eagle-service.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/eagle-service.conf b/eagle-assembly/src/main/conf/eagle-service.conf deleted file mode 100644 index 229458c..0000000 --- a/eagle-assembly/src/main/conf/eagle-service.conf +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# eagle configuration -eagle{ - # eagle web service configuration - service{ - # storage type: ["hbase","jdbc"] - # default is "hbase" - storage-type="hbase" - - # hbase configuration: hbase.zookeeper.quorum - # default is "localhost" - hbase-zookeeper-quorum="localhost" - - # hbase configuration: hbase.zookeeper.property.clientPort - # default is 2181 - hbase-zookeeper-property-clientPort=2181 - - # hbase configuration: zookeeper.znode.parent - # default is "/hbase" - zookeeper-znode-parent="/hbase-unsecure" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/log4j.properties b/eagle-assembly/src/main/conf/log4j.properties deleted file mode 100644 index 25331ab..0000000 --- a/eagle-assembly/src/main/conf/log4j.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -log4j.rootLogger=INFO, stdout - - eagle.log.dir=../logs - eagle.log.file=eagle.log - -# standard output -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n - -# Daily Rolling File Appender - log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender - log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file} - log4j.appender.DRFA.DatePattern=.yyyy-MM-dd -## 30-day backup -# log4j.appender.DRFA.MaxBackupIndex=30 - log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/pipeline.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/pipeline.conf b/eagle-assembly/src/main/conf/pipeline.conf deleted file mode 100644 index c1c7758..0000000 --- a/eagle-assembly/src/main/conf/pipeline.conf +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig": { - "env" : "storm" - "mode" : "cluster" - "nimbusHost":"sandbox.hortonworks.com", - "nimbusThriftPort":6627 - } - "eagleProps" : { - "dataJoinPollIntervalSec" : 30 - "mailHost" : "smtp.server.host" - "mailSmtpPort":"25" - "mailDebug" : "true" - "eagleService": { - "host": "localhost" - "port": 9099 - "username": "admin" - "password": "secret" - } - } - "dynamicConfigSource" : { - "enabled" : true - "initDelayMillis" : 0 - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf b/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf deleted file mode 100644 index 3f14c47..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - config { - envContextConfig { - "topologyName" : "sandbox-hadoopjmx-pipeline" - } - eagleProps { - "site" : "sandbox" - "application": "HADOOP" - } - } - - dataflow { - KafkaSource.hadoopNNJmxStream { - parallism = 1000 - topic = "nn_jmx_metric_sandbox" - zkConnection = "sandbox.hortonworks.com:2181" - zkConnectionTimeoutMS = 15000 - consumerGroupId = "Consumer" - fetchSize = 1048586 - transactionZKServers = "sandbox.hortonworks.com" - transactionZKPort = 2181 - transactionZKRoot = "/consumers" - transactionStateUpdateMS = 2000 - deserializerClass = "org.apache.eagle.datastream.storm.JsonMessageDeserializer" - } - - Alert.hadoopNNJmxStreamAlertExecutor { - upStreamNames = [hadoopNNJmxStream] - alertExecutorId = hadoopNNJmxStreamAlertExecutor - } - - hadoopNNJmxStream -> hadoopNNJmxStreamAlertExecutor{} - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf b/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf deleted file mode 100644 index d91ef9f..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "cluster", - "topologyName" : "hadoopJmxMetricTopology", - "stormConfigFile" : "hadoopjmx.yaml", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "hadoopJmxMetricAlertExecutor*" : 1 - } - }, - "dataSourceConfig": { - "topic" : "nn_jmx_metric_sandbox", - "zkConnection" : "sandbox.hortonworks.com:2181", - "zkConnectionTimeoutMS" : 15000, - "consumerGroupId" : "EagleConsumer", - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.datastream.storm.JsonMessageDeserializer", - "transactionZKServers" : "sandbox.hortonworks.com", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/consumers", - "transactionStateUpdateMS" : 2000 - }, - "alertExecutorConfigs" : { - "hadoopJmxMetricAlertExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner" - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "hadoopJmxMetricDataSource", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "balancePartitionEnabled" : true, - #"partitionRefreshIntervalInMin" : 60, - #"kafkaStatisticRangeInMin" : 60, - "eagleService": { - "host": "localhost", - "port": 9099, - "username": "admin", - "password": "secret" - } - "readHdfsUserCommandPatternFrom" : "file" - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hbaseSecurityLog-application.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hbaseSecurityLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hbaseSecurityLog-application.conf deleted file mode 100644 index 58468ea..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hbaseSecurityLog-application.conf +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "cluster", - "topologyName" : "sandbox-hbaseSecurityLog-topology", - "stormConfigFile" : "security-auditlog-storm.yaml", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "hbaseSecurityLogAlertExecutor*" : 1 - } - }, - "dataSourceConfig": { - "topic" : "sandbox_hbase_security_log", - "zkConnection" : "127.0.0.1:2181", - "zkConnectionTimeoutMS" : 15000, - "consumerGroupId" : "EagleConsumer", - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer", - "transactionZKServers" : "127.0.0.1", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/consumers", - "consumerGroupId" : "eagle.hbasesecurity.consumer", - "transactionStateUpdateMS" : 2000 - }, - "alertExecutorConfigs" : { - "hbaseSecurityLogAlertExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner" - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "hbaseSecurityLog", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "eagleService": { - "host": "localhost", - "port": 9099 - "username": "admin", - "password": "secret" - } - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hdfsAuditLog-application.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hdfsAuditLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hdfsAuditLog-application.conf deleted file mode 100644 index 4634749..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hdfsAuditLog-application.conf +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "cluster", - "topologyName" : "sandbox-hdfsAuditLog-topology", - "stormConfigFile" : "security-auditlog-storm.yaml", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "hdfsAuditLogAlertExecutor*" : 1 - } - }, - "dataSourceConfig": { - "topic" : "sandbox_hdfs_audit_log", - "zkConnection" : "127.0.0.1:2181", - "brokerZkPath" : "/brokers", - "zkConnectionTimeoutMS" : 15000, - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.security.auditlog.HdfsAuditLogKafkaDeserializer", - "transactionZKServers" : "127.0.0.1", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/consumers", - "consumerGroupId" : "eagle.hdfsaudit.consumer", - "transactionStateUpdateMS" : 2000 - }, - "alertExecutorConfigs" : { - "hdfsAuditLogAlertExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner", - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "hdfsAuditLog", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "eagleService": { - "host": "localhost", - "port": 9099 - "username": "admin", - "password": "secret" - } - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf deleted file mode 100644 index d54e123..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "cluster", - "topologyName" : "sandbox-hdfsSecurityLog-topology", - "stormConfigFile" : "security-auditlog-storm.yaml", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "hdfsSecurityLogAlertExecutor*" : 1 - } - }, - "dataSourceConfig": { - "topic" : "sandbox_hdfs_security_log", - "zkConnection" : "127.0.0.1:2181", - "brokerZkPath" : "/brokers", - "zkConnectionTimeoutMS" : 15000, - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer", - "transactionZKServers" : "127.0.0.1", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/consumers", - "consumerGroupId" : "eagle.hdfssecurity.consumer", - "transactionStateUpdateMS" : 2000 - }, - "alertExecutorConfigs" : { - "hdfsSecurityLogAlertExecutor" : { - "parallelism" : 1, - "partitioner" : "eagle.alert.policy.DefaultPolicyPartitioner", - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "hdfsSecurityLog", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "eagleService": { - "host": "localhost", - "port": 9099 - "username": "admin", - "password": "secret" - } - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-hiveQueryLog-application.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-hiveQueryLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hiveQueryLog-application.conf deleted file mode 100644 index 6601aff..0000000 --- a/eagle-assembly/src/main/conf/sandbox-hiveQueryLog-application.conf +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "cluster", - "topologyName" : "sandbox-hiveQueryRunning-topology", - "stormConfigFile" : "hive.storm.yaml", - "parallelismConfig" : { - "msgConsumer" : 2 - } - }, - "dataSourceConfig": { - "flavor" : "stormrunning", - "zkQuorum" : "localhost:2181", - "zkRoot" : "/jobrunning", - "zkSessionTimeoutMs" : 15000, - "zkRetryTimes" : 3, - "zkRetryInterval" : 2000, - "RMEndPoints" : "http://localhost:8088/";, - "HSEndPoint" : "http://localhost:19888/";, - "partitionerCls" : "org.apache.eagle.job.DefaultJobPartitionerImpl" - }, - "eagleProps" : { - "site" : "sandbox", - "application" : "hiveQueryLog", - "mailHost" : "atom.xyz.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "eagleService": { - "host": "localhost", - "port": 9099, - "username": "admin", - "password": "secret" - } - }, - "alertExecutorConfigs" : { - "hiveAccessAlertByRunningJob" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner", - "needValidation" : "true" - } - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000, - "ignoreDeleteFromSource" : true - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf b/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf deleted file mode 100644 index fd8dd56..0000000 --- a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# eagle configuration -eagle{ - # eagle site id - site = "sandbox" - - # eagle service configuration - service { - # eagle service host, default: "localhost" - host = "localhost" - # eagle service port, default: 8080 - port = 9099 - username = "admin" - password = "secret" - } - - # eagle userprofile configuration - userprofile { - # training audit log input path - training-audit-path = "file:///usr/hdp/2.3.0.0-2130/eagle/lib/userprofile/data/*.txt" - - # detection audit log input path - detection-audit-path = "file:///usr/hdp/2.3.0.0-2130/eagle/lib/userprofile/data/validate" - - # detection output kafka brokers - # default: localhost:9200 - detection-kafka-brokers = "sandbox.hortonworks.com:6667" - - # detection output kafka topic - detection-kafka-topic = "sandbox_hdfs_audit_log" - } -} - -akka { - # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs - # to STDOUT) - loggers = ["akka.event.slf4j.Slf4jLogger"] - - # Log level used by the configured loggers (see "loggers") as soon - # as they have been started; before that, see "stdout-loglevel" - # Options: OFF, ERROR, WARNING, INFO, DEBUG - loglevel = "INFO" - - # Log level for the very basic logger activated during ActorSystem startup. - # This logger prints the log messages to stdout (System.out). - # Options: OFF, ERROR, WARNING, INFO, DEBUG - stdout-loglevel = "INFO" - - # Filter of log events that is used by the LoggingAdapter before - # publishing log events to the eventStream. - logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/conf/sandbox-userprofile-topology.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/sandbox-userprofile-topology.conf b/eagle-assembly/src/main/conf/sandbox-userprofile-topology.conf deleted file mode 100644 index abe8cb8..0000000 --- a/eagle-assembly/src/main/conf/sandbox-userprofile-topology.conf +++ /dev/null @@ -1,49 +0,0 @@ -{ - "deployInstanceIdentifier" : { - "programId" : "mlProgramId" - }, - "envContextConfig" : { - "env" : "storm", - "topologyName" : "sandbox-userprofile-topology", - "mode" : "cluster", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "userProfileAnomalyDetectionExecutor*" : 1 - }, - "stormConfigFile":"userprofile.storm.yaml" - }, - "dataSourceConfig": { - "topic" : "sandbox_hdfs_audit_log", - "zkConnection" : "localhost:2181", - "zkConnectionTimeoutMS" : 15000, - "consumerGroupId" : "eagle.userprofile.consumer", - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.security.auditlog.HdfsAuditLogKafkaDeserializer", - "transactionZKServers" : "localhost", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/brokers/topics", - "transactionStateUpdateMS" : 2000 - }, - "eagleProps" : { - "site":"sandbox", - "application":"userProfile", - "eagleService": { - "host": "localhost", - "port": "9099", - "username": "admin", - "password": "secret" - } - }, - "alertExecutorConfigs" : { - "userProfileAnomalyDetectionExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner", - "needValidation":true - } - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 5000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/docs/kafka.rb ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/docs/kafka.rb b/eagle-assembly/src/main/docs/kafka.rb deleted file mode 100644 index 4c5f9b3..0000000 --- a/eagle-assembly/src/main/docs/kafka.rb +++ /dev/null @@ -1,191 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'logstash/namespace' -require 'logstash/outputs/base' -require 'jruby-kafka' - -# Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on -# the broker. -# -# The only required configuration is the topic name. The default codec is json, -# so events will be persisted on the broker in json format. If you select a codec of plain, -# Logstash will encode your messages with not only the message but also with a timestamp and -# hostname. If you do not want anything but your message passing through, you should make the output -# configuration something like: -# [source,ruby] -# output { -# kafka { -# codec => plain { -# format => "%{message}" -# } -# } -# } -# For more information see http://kafka.apache.org/documentation.html#theproducer -# -# Kafka producer configuration: http://kafka.apache.org/documentation.html#producerconfigs -class LogStash::Outputs::Kafka < LogStash::Outputs::Base - config_name 'kafka' - milestone 2 - - default :codec, 'json' - # This is for bootstrapping and the producer will only use it for getting metadata (topics, - # partitions and replicas). The socket connections for sending the actual data will be - # established based on the broker information returned in the metadata. The format is - # `host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a - # subset of brokers. - config :broker_list, :validate => :string, :default => 'localhost:9092' - # The topic to produce the messages to - config :topic_id, :validate => :string, :required => true - # This parameter allows you to specify the compression codec for all data generated by this - # producer. Valid values are `none`, `gzip` and `snappy`. - config :compression_codec, :validate => %w( none gzip snappy ), :default => 'none' - # This parameter allows you to set whether compression should be turned on for particular - # topics. If the compression codec is anything other than `NoCompressionCodec`, - # enable compression only for specified topics if any. If the list of compressed topics is - # empty, then enable the specified compression codec for all topics. If the compression codec - # is `NoCompressionCodec`, compression is disabled for all topics - config :compressed_topics, :validate => :string, :default => '' - # This value controls when a produce request is considered completed. Specifically, - # how many other brokers must have committed the data to their log and acknowledged this to the - # leader. For more info, see -- http://kafka.apache.org/documentation.html#producerconfigs - config :request_required_acks, :validate => [-1,0,1], :default => 0 - # The serializer class for messages. The default encoder takes a byte[] and returns the same byte[] - config :serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder' - # The partitioner class for partitioning messages amongst partitions in the topic. The default - # partitioner is based on the hash of the key. If the key is null, - # the message is sent to a random partition in the broker. - # NOTE: `topic_metadata_refresh_interval_ms` controls how long the producer will distribute to a - # partition in the topic. This defaults to 10 mins, so the producer will continue to write to a - # single partition for 10 mins before it switches - config :partitioner_class, :validate => :string, :default => 'kafka.producer.DefaultPartitioner' - # The amount of time the broker will wait trying to meet the `request.required.acks` requirement - # before sending back an error to the client. - config :request_timeout_ms, :validate => :number, :default => 10000 - # This parameter specifies whether the messages are sent asynchronously in a background thread. - # Valid values are (1) async for asynchronous send and (2) sync for synchronous send. By - # setting the producer to async we allow batching together of requests (which is great for - # throughput) but open the possibility of a failure of the client machine dropping unsent data. - config :producer_type, :validate => %w( sync async ), :default => 'sync' - # The serializer class for keys (defaults to the same as for messages if nothing is given) - config :key_serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder' - # This property will cause the producer to automatically retry a failed send request. This - # property specifies the number of retries when such failures occur. Note that setting a - # non-zero value here can lead to duplicates in the case of network errors that cause a message - # to be sent but the acknowledgement to be lost. - config :message_send_max_retries, :validate => :number, :default => 3 - # Before each retry, the producer refreshes the metadata of relevant topics to see if a new - # leader has been elected. Since leader election takes a bit of time, - # this property specifies the amount of time that the producer waits before refreshing the - # metadata. - config :retry_backoff_ms, :validate => :number, :default => 100 - # The producer generally refreshes the topic metadata from brokers when there is a failure - # (partition missing, leader not available...). It will also poll regularly (default: every - # 10min so 600000ms). If you set this to a negative value, metadata will only get refreshed on - # failure. If you set this to zero, the metadata will get refreshed after each message sent - # (not recommended). Important note: the refresh happen only AFTER the message is sent, - # so if the producer never sends a message the metadata is never refreshed - config :topic_metadata_refresh_interval_ms, :validate => :number, :default => 600 * 1000 - # Maximum time to buffer data when using async mode. For example a setting of 100 will try to - # batch together 100ms of messages to send at once. This will improve throughput but adds - # message delivery latency due to the buffering. - config :queue_buffering_max_ms, :validate => :number, :default => 5000 - # The maximum number of unsent messages that can be queued up the producer when using async - # mode before either the producer must be blocked or data must be dropped. - config :queue_buffering_max_messages, :validate => :number, :default => 10000 - # The amount of time to block before dropping messages when running in async mode and the - # buffer has reached `queue.buffering.max.messages`. If set to 0 events will be enqueued - # immediately or dropped if the queue is full (the producer send call will never block). If set - # to -1 the producer will block indefinitely and never willingly drop a send. - config :queue_enqueue_timeout_ms, :validate => :number, :default => -1 - # The number of messages to send in one batch when using async mode. The producer will wait - # until either this number of messages are ready to send or `queue.buffer.max.ms` is reached. - config :batch_num_messages, :validate => :number, :default => 200 - # Socket write buffer size - config :send_buffer_bytes, :validate => :number, :default => 100 * 1024 - # The client id is a user-specified string sent in each request to help trace calls. It should - # logically identify the application making the request. - config :client_id, :validate => :string, :default => '' - # Provides a way to specify a partition key as a string. To specify a partition key for - # Kafka, configure a format that will produce the key as a string. Defaults - # `key_serializer_class` to `kafka.serializer.StringEncoder` to match. For example, to partition - # by host: - # [source,ruby] - # output { - # kafka { - # partition_key_format => "%{host}" - # } - # } - config :partition_key_format, :validate => :string, :default => nil - - public - def register - LogStash::Logger.setup_log4j(@logger) - - options = { - :broker_list => @broker_list, - :compression_codec => @compression_codec, - :compressed_topics => @compressed_topics, - :request_required_acks => @request_required_acks, - :serializer_class => @serializer_class, - :partitioner_class => @partitioner_class, - :request_timeout_ms => @request_timeout_ms, - :producer_type => @producer_type, - :key_serializer_class => @key_serializer_class, - :message_send_max_retries => @message_send_max_retries, - :retry_backoff_ms => @retry_backoff_ms, - :topic_metadata_refresh_interval_ms => @topic_metadata_refresh_interval_ms, - :queue_buffering_max_ms => @queue_buffering_max_ms, - :queue_buffering_max_messages => @queue_buffering_max_messages, - :queue_enqueue_timeout_ms => @queue_enqueue_timeout_ms, - :batch_num_messages => @batch_num_messages, - :send_buffer_bytes => @send_buffer_bytes, - :client_id => @client_id - } - @producer = Kafka::Producer.new(options) - @producer.connect - - @logger.info('Registering kafka producer', :topic_id => @topic_id, :broker_list => @broker_list) - - @codec.on_event do |data| - begin - @producer.send_msg(@current_topic_id,@partition_key,data) - rescue LogStash::ShutdownSignal - @logger.info('Kafka producer got shutdown signal') - rescue => e - @logger.warn('kafka producer threw exception, restarting', - :exception => e) - end - end - end # def register - - def receive(event) - return unless output?(event) - if event == LogStash::SHUTDOWN - finished - return - end - @partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end - @current_topic_id = if @topic_id.nil? then nil else event.sprintf(@topic_id) end - @codec.encode(event) - @partition_key = nil - @current_topic_id = nil - end - - def teardown - @producer.close - end -end #class LogStash::Outputs::Kafka - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/docs/logstash-kafka-conf.md ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/docs/logstash-kafka-conf.md b/eagle-assembly/src/main/docs/logstash-kafka-conf.md deleted file mode 100644 index 84952e0..0000000 --- a/eagle-assembly/src/main/docs/logstash-kafka-conf.md +++ /dev/null @@ -1,164 +0,0 @@ -<!-- -{% comment %} -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -{% endcomment %} ---> - -# Logstash-kafka - -### Install logstash-kafka plugin - - -#### For Logstash 1.5.x - - -logstash-kafka has been intergrated into [logstash-input-kafka][logstash-input-kafka] and [logstash-output-kafka][logstash-output-kafka], you can directly use it. - -[logstash-input-kafka]: https://github.com/logstash-plugins/logstash-input-kafka -[logstash-output-kafka]: https://github.com/logstash-plugins/logstash-output-kafka - -#### For Logstash 1.4.x - -In logstash 1.4.x, the online version does not support specifying partition\_key for Kafka producer, and data will be produced into each partitions in turn. For eagle, we need to use the src in hdfs\_audit\_log as the partition key, so some hacking work have been done. If you have the same requirment, you can follow it. - -1. Install logstash-kafka - - cd /path/to/logstash - GEM_HOME=vendor/bundle/jruby/1.9 GEM_PATH= java -jar vendor/jar/jruby-complete-1.7.11.jar -S gem install logstash-kafka - cp -R vendor/bundle/jruby/1.9/gems/logstash-kafka-*-java/{lib/logstash/*,spec/*} {lib/logstash/,spec/} - # test install - USE_JRUBY=1 bin/logstash rspec spec/**/kafka*.rb - - or - - cd /path/to/logstash-kafka - make tarball - <!-- a tarball package will be generated under build, including logstash --> - -2. Hacking the kafka.rb - - We have added partition\_key\_format, which is used to specify the partition_key and supported by logstash 1.5.x, into lib/logstash/outputs/kafka.rb. More details are shown [here](https://github.xyz.com/eagle/eagle/blob/master/eagle-assembly/src/main/docs/kafka.rb). - - config :partition_key_format, :validate => :string, :default => nil - - public - def register - LogStash::Logger.setup_log4j(@logger) - - options = { - :broker_list => @broker_list, - :compression_codec => @compression_codec, - :compressed_topics => @compressed_topics, - :request_required_acks => @request_required_acks, - :serializer_class => @serializer_class, - :partitioner_class => @partitioner_class, - :request_timeout_ms => @request_timeout_ms, - :producer_type => @producer_type, - :key_serializer_class => @key_serializer_class, - :message_send_max_retries => @message_send_max_retries, - :retry_backoff_ms => @retry_backoff_ms, - :topic_metadata_refresh_interval_ms => @topic_metadata_refresh_interval_ms, - :queue_buffering_max_ms => @queue_buffering_max_ms, - :queue_buffering_max_messages => @queue_buffering_max_messages, - :queue_enqueue_timeout_ms => @queue_enqueue_timeout_ms, - :batch_num_messages => @batch_num_messages, - :send_buffer_bytes => @send_buffer_bytes, - :client_id => @client_id - } - @producer = Kafka::Producer.new(options) - @producer.connect - - @logger.info('Registering kafka producer', :topic_id => @topic_id, :broker_list => @broker_list) - - @codec.on_event do |data| - begin - @producer.send_msg(@current_topic_id,@partition_key,data) - rescue LogStash::ShutdownSignal - @logger.info('Kafka producer got shutdown signal') - rescue => e - @logger.warn('kafka producer threw exception, restarting', - :exception => e) - end - end - end # def register - - def receive(event) - return unless output?(event) - if event == LogStash::SHUTDOWN - finished - return - end - @partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end - @current_topic_id = if @topic_id.nil? then nil else event.sprintf(@topic_id) end - @codec.encode(event) - @partition_key = nil - @current_topic_id = nil - end - - -### Create logstash configuration file -Go to the logstash root dir, and create a configure file - - input { - file { - type => "hdp-nn-audit" - path => "/path/to/audit.log" - start_position => end - sincedb_path => "/var/log/logstash/" - } - } - - filter{ - if [type] == "hdp-nn-audit" { - grok { - match => ["message", "ugi=(?<user>([\w\d\-]+))@|ugi=(?<user>([\w\d\-]+))/[\w\d\-.]+@|ugi=(?<user>([\w\d.\-_]+))[\s(]+"] - } - } - } - - output { - if [type] == "hdp-nn-audit" { - kafka { - codec => plain { - format => "%{message}" - } - broker_list => "localhost:9092" - topic_id => "hdfs_audit_log" - request_required_acks => 0 - request_timeout_ms => 10000 - producer_type => "async" - message_send_max_retries => 3 - retry_backoff_ms => 100 - queue_buffering_max_ms => 5000 - queue_enqueue_timeout_ms => 5000 - batch_num_messages => 200 - send_buffer_bytes => 102400 - client_id => "hdp-nn-audit" - partition_key_format => "%{user}" - } - # stdout { codec => rubydebug } - } - } - -#### grok pattern testing -We have 3 typical patterns for ugi field as follows -2015-02-11 15:00:00,000 INFO FSNamesystem.audit: allowed=true ugi=us...@xyz.com (auth:TOKEN) ip=/10.115.44.55 cmd=open src=/apps/hdmi-technology/b_pulsar_coe/schema/avroschema/Session.avsc dst=null perm=null -2015-02-11 15:00:00,000 INFO FSNamesystem.audit: allowed=true ugi=hdc_uc4_platform (auth:TOKEN) via sg_...@xyz.com (auth:TOKEN) ip=/10.115.11.54 cmd=open src=/sys/soj/event/2015/02/08/same_day/00000000000772509716119204458864#3632400774990000-949461-r-01459.avro dst=null perm=null - -### Reference Links -1. [logstash-kafka](https://github.com/joekiller/logstash-kafka) -2. [logstash](https://github.com/elastic/logstash) - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/examples/eagle-sandbox-starter.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/examples/eagle-sandbox-starter.sh b/eagle-assembly/src/main/examples/eagle-sandbox-starter.sh deleted file mode 100644 index 286bc87..0000000 --- a/eagle-assembly/src/main/examples/eagle-sandbox-starter.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/../bin/eagle-env.sh -eagle_bin=$EAGLE_HOME/bin - - -################################################################### -# STEP 1: Check Installation -################################################################### - -echo "STEP [1/3]: checking environment" -$eagle_bin/eagle-check-env.sh -[ $? != 0 ] && exit 1 - -pid_dir=/var/run - -# Check HBase if it has been started -hbase_master_pid=${pid_dir}/hbase/hbase-hbase-master.pid -hbase_regionserver_pid=${pid_dir}/hbase/hbase-hbase-regionserver.pid -echo "Checking if hbase is running ..." - -if [ -f $hbase_master_pid ] && \ - ps aux | grep -v grep | grep $(cat $hbase_master_pid) > /dev/null -then - echo "HBase Master is running as process `cat $hbase_master_pid`." -else - echo "Error: HBase Master is not running. Please start it via Ambari." - exit 1 -fi - -if [ -f $hbase_regionserver_pid ] && \ - ps aux | grep -v grep | grep $(cat $hbase_regionserver_pid) > /dev/null -then - echo "HBase RegionServer is running as process `cat $hbase_regionserver_pid`." -else - echo "Error: HBase RegionServer is not running. Please start it via Ambari." - exit 1 -fi - -# Check kafka if it has been started -kafka_pid=$pid_dir/kafka/kafka.pid -echo "Checking if kafka is running ..." - -if [ -f $kafka_pid ] && ps aux | grep -v grep | grep $(cat $kafka_pid) > /dev/null -then - echo "Kafka is running as process `cat $kafka_pid`." -else - echo "Error: Kafka is not running. Please start it via Ambari." - exit 1 -fi - -# Check storm if it has been started -nimbus_pid=$pid_dir/storm/nimbus.pid -supervisor_pid=$pid_dir/storm/supervisor.pid -ui_pid=$pid_dir/storm/ui.pid -echo "Checking if storm is running ..." - -if ! ([ -f $nimbus_pid ] && ps aux | grep -v grep | grep $(cat $nimbus_pid) > /dev/null) -then - echo "Error: Storm Nimbus is not running" - exit 1 -fi - -if ! ([ -f $supervisor_pid ] && ps aux | grep -v grep | grep $(cat $supervisor_pid) > /dev/null) -then - echo "Error: Storm Supervisor is not running" - exit 1 -fi - -if ! ([ -f $ui_pid ] && ps aux | grep -v grep | grep $(cat $ui_pid) > /dev/null) -then - echo "Error: Storm UI is not running" - exit 1 -fi - -echo "Storm is running" - - -################################################################### -# STEP 2: Starting Eagle Service -################################################################### - -echo "STEP [2/3]: start eagle service" -$eagle_bin/eagle-service.sh start - - -################################################################### -# STEP 3: Starting Eagle Topology -################################################################### - -echo "STEP [3/3]: start eagle topology" -$eagle_bin/eagle-service-init.sh -[ $? != 0 ] && exit 1 - -echo "Creating kafka topics for eagle ... " -KAFKA_HOME=/usr/hdp/current/kafka-broker -EAGLE_ZOOKEEPER_QUORUM=localhost:2181 -topic=`${KAFKA_HOME}/bin/kafka-topics.sh --list --zookeeper $EAGLE_ZOOKEEPER_QUORUM --topic sandbox_hdfs_audit_log` -if [ -z $topic ]; then - $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $EAGLE_ZOOKEEPER_QUORUM --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log -fi - -if [ $? = 0 ]; then -echo "==> Create kafka topic successfully for eagle" -else -echo "==> Failed, exiting" -exit 1 -fi -$eagle_bin/eagle-topology-init.sh -[ $? != 0 ] && exit 1 -${EAGLE_HOME}/examples/sample-sensitivity-resource-create.sh -[ $? != 0 ] && exit 1 -${EAGLE_HOME}/examples/sample-policy-create.sh -[ $? != 0 ] && exit 1 -$eagle_bin/eagle-topology.sh --main org.apache.eagle.security.auditlog.HdfsAuditLogProcessorMain --config ${EAGLE_HOME}/conf/sandbox-hdfsAuditLog-application.conf start -[ $? != 0 ] && exit 1 -$eagle_bin/eagle-topology.sh --main org.apache.eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain --config ${EAGLE_HOME}/conf/sandbox-hiveQueryLog-application.conf start -[ $? != 0 ] && exit 1 -$eagle_bin/eagle-topology.sh --main org.apache.eagle.security.userprofile.UserProfileDetectionMain --config ${EAGLE_HOME}/conf/sandbox-userprofile-topology.conf start -[ $? != 0 ] && exit 1 - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh b/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh deleted file mode 100644 index 9e9e861..0000000 --- a/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with` -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/../bin/eagle-env.sh - -##### add policies ########## -echo "" -echo "Importing policy: safeModePolicy " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"; \ - -d ' - [ - { - "prefix": "alertdef", - "tags": { - "site": "sandbox", - "application": "hadoopJmxMetricDataSource", - "policyId": "safeModePolicy", - "alertExecutorId": "hadoopJmxMetricAlertExecutor", - "policyType": "siddhiCEPEngine" - }, - "description": "jmx metric ", - "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[component==\\\"namenode\\\" and metric == \\\"hadoop.namenode.fsnamesystemstate.fsstate\\\" and convert(value, \\\"long\\\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}", - "enabled": true, - "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}", - "notificationDef": "[{\"sender\":\"ea...@apache.org\",\"recipients\":\"ea...@apache.org\",\"subject\":\"missing block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]" - } - ] - ' - -exit 0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh b/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh deleted file mode 100644 index 8f227b0..0000000 --- a/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/../bin/eagle-env.sh -eagle_bin=$EAGLE_HOME/bin - - -################################################################### -# STEP 1: Check Installation -################################################################### - -echo "STEP [1/3]: checking environment" -$eagle_bin/eagle-check-env.sh -[ $? != 0 ] && exit 1 - -pid_dir=/var/run - -# Check HBase if it has been started -hbase_master_pid=${pid_dir}/hbase/hbase-hbase-master.pid -hbase_regionserver_pid=${pid_dir}/hbase/hbase-hbase-regionserver.pid -echo "Checking if hbase is running ..." - -if [ -f $hbase_master_pid ] && \ - ps aux | grep -v grep | grep $(cat $hbase_master_pid) > /dev/null -then - echo "HBase Master is running as process `cat $hbase_master_pid`." -else - echo "Error: HBase Master is not running. Please start it via Ambari." - exit 1 -fi - -if [ -f $hbase_regionserver_pid ] && \ - ps aux | grep -v grep | grep $(cat $hbase_regionserver_pid) > /dev/null -then - echo "HBase RegionServer is running as process `cat $hbase_regionserver_pid`." -else - echo "Error: HBase RegionServer is not running. Please start it via Ambari." - exit 1 -fi - -# Check kafka if it has been started -kafka_pid=$pid_dir/kafka/kafka.pid -echo "Checking if kafka is running ..." - -if [ -f $kafka_pid ] && ps aux | grep -v grep | grep $(cat $kafka_pid) > /dev/null -then - echo "Kafka is running as process `cat $kafka_pid`." -else - echo "Error: Kafka is not running. Please start it via Ambari." - exit 1 -fi - -# Check storm if it has been started -nimbus_pid=$pid_dir/storm/nimbus.pid -supervisor_pid=$pid_dir/storm/supervisor.pid -ui_pid=$pid_dir/storm/ui.pid -echo "Checking if storm is running ..." - -if ! ([ -f $nimbus_pid ] && ps aux | grep -v grep | grep $(cat $nimbus_pid) > /dev/null) -then - echo "Error: Storm Nimbus is not running" - exit 1 -fi - -if ! ([ -f $supervisor_pid ] && ps aux | grep -v grep | grep $(cat $supervisor_pid) > /dev/null) -then - echo "Error: Storm Supervisor is not running" - exit 1 -fi - -if ! ([ -f $ui_pid ] && ps aux | grep -v grep | grep $(cat $ui_pid) > /dev/null) -then - echo "Error: Storm UI is not running" - exit 1 -fi - -echo "Storm is running" - - -################################################################### -# STEP 2: Starting Eagle Service -################################################################### - -echo "STEP [2/3]: Start Eagle Service" -$eagle_bin/eagle-service.sh start - - -################################################################### -# STEP 3: Starting Eagle Topology -################################################################### - -echo "STEP [3/3]: Init Eagle Service" -$eagle_bin/eagle-service-init.sh -[ $? != 0 ] && exit 1 - -echo "Creating kafka topics for eagle ... " -KAFKA_HOME=/usr/hdp/current/kafka-broker -EAGLE_ZOOKEEPER_QUORUM=localhost:2181 -topic=`${KAFKA_HOME}/bin/kafka-topics.sh --list --zookeeper $EAGLE_ZOOKEEPER_QUORUM --topic nn_jmx_metric_sandbox` -if [ -z $topic ]; then - $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $EAGLE_ZOOKEEPER_QUORUM --replication-factor 1 --partitions 1 --topic nn_jmx_metric_sandbox -fi - -if [ $? = 0 ]; then -echo "==> Create kafka topic successfully for Hadoop Metric Monitoring" -else -echo "==> Failed, exiting" -exit 1 -fi -$eagle_bin/hadoop-metric-monitor.sh -[ $? != 0 ] && exit 1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/examples/sample-policy-create.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/examples/sample-policy-create.sh b/eagle-assembly/src/main/examples/sample-policy-create.sh deleted file mode 100644 index 71e099e..0000000 --- a/eagle-assembly/src/main/examples/sample-policy-create.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/../bin/eagle-env.sh - -su hdfs -c "hdfs dfs -touchz /tmp/private" -#su hdfs -c "hdfs dfs -touchz /tmp/sensitive" - -#### create hdfs policy sample in sandbox -echo "create hdfs policy sample in sandbox... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"; -d \ -'[{"tags":{"site":"sandbox","application":"hdfsAuditLog","alertExecutorId":"hdfsAuditLogAlertExecutor","policyId":"viewPrivate","policyType":"siddhiCEPEngine"},"desc":"view private file","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from hdfsAuditLogEventStream[(cmd=='\'open\'') and (src=='\'/tmp/private\'')] select * insert into outputStream\"}","dedupeDef": "{\"alertDedupIntervalMin\":0,\"emailDedupIntervalMin\":1440}","notificationDef": "[{\"subject\":\"just for test\",\"sender\":\"nob...@test.com\",\"recipients\":\"nob...@test.com\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]","remediationDef":"","enabled":true}]' - -#### create hive policy sample in sandbox -echo "create hive policy sample in sandbox... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"; -d \ -'[{"tags":{"site":"sandbox","application":"hiveQueryLog","alertExecutorId":"hiveAccessAlertByRunningJob","policyId":"queryPhoneNumber","policyType":"siddhiCEPEngine"},"desc":"query sensitive hive data","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from hiveAccessLogStream[(sensitivityType=='\'PHONE_NUMBER\'')] select * insert into outputStream;\"}","dedupeDef": "{\"alertDedupIntervalMin\":0,\"emailDedupIntervalMin\":1440}","notificationDef": "[{\"subject\":\"just for test\",\"sender\":\"nob...@test.com\",\"recipients\":\"nob...@test.com\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]","remediationDef":"","enabled":"true"}]' http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/examples/sample-sensitivity-resource-create.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/examples/sample-sensitivity-resource-create.sh b/eagle-assembly/src/main/examples/sample-sensitivity-resource-create.sh deleted file mode 100644 index 4a8f06f..0000000 --- a/eagle-assembly/src/main/examples/sample-sensitivity-resource-create.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/../bin/eagle-env.sh - -#### create hive sensitivity sample in sandbox -echo "create hive sensitivity sample in sandbox... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=HiveResourceSensitivityService"; -d '[{"tags":{"site" : "sandbox", "hiveResource":"/xademo/customer_details/phone_number"}, "sensitivityType": "PHONE_NUMBER"}]' - - -#### create hdfs sensitivity sample in sandbox -echo "create hdfs sensitivity sample in sandbox... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FileSensitivityService"; -d '[{"tags":{"site" : "sandbox", "filedir":"/tmp/private"}, "sensitivityType": "PRIVATE"}]' - -#### create hbase sensitivity sample in sandbox -echo "create hdfs sensitivity sample in sandbox... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=HbaseResourceSensitivityService"; -d '[{"tags":{"site":"sandbox","hbaseResource":"default:alertStreamSchema"},"sensitivityType":"PrivateTable"}]' - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-assembly/src/main/lib/share/.placeholder ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/lib/share/.placeholder b/eagle-assembly/src/main/lib/share/.placeholder deleted file mode 100644 index 823c394..0000000 --- a/eagle-assembly/src/main/lib/share/.placeholder +++ /dev/null @@ -1,14 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. \ No newline at end of file
