Repository: incubator-eagle Updated Branches: refs/heads/develop 0f7ddab58 -> 0f11a591a
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/application-plugintest.conf ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/application-plugintest.conf b/eagle-examples/eagle-topology-example/src/main/resources/application-plugintest.conf deleted file mode 100644 index b35abc8..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/application-plugintest.conf +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "local", - "topologyName" : "pluginTest", - "stormConfigFile" : "persist-test-storm.yaml", - "parallelismConfig" : { - "testSpout" : 1, - "testExecutor*" : 1 - } - }, - "dataSourceConfig": { - }, - "alertExecutorConfigs" : { - "testExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner" - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "testSpout", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "balancePartitionEnabled" : true, - #"partitionRefreshIntervalInMin" : 60, - #"kafkaStatisticRangeInMin" : 60, - "eagleService": { - "host": "localhost", - "port": 9099, - "username": "admin", - "password": "secret" - }, - "readHdfsUserCommandPatternFrom" : "file" - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/application.conf b/eagle-examples/eagle-topology-example/src/main/resources/application.conf deleted file mode 100644 index 9629917..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/application.conf +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "envContextConfig" : { - "env" : "storm", - "mode" : "local", - "topologyName" : "persistTestTopology", - "stormConfigFile" : "persist-test-storm.yaml", - "parallelismConfig" : { - "kafkaMsgConsumer" : 1, - "hdfsAuditLogAlertExecutor*" : 1 - } - }, - "dataSourceConfig": { - "topic" : "persist_test_log", - "zkConnection" : "localhost:2181", - "zkConnectionTimeoutMS" : 15000, - "consumerGroupId" : "EagleConsumer", - "fetchSize" : 1048586, - "deserializerClass" : "org.apache.eagle.example.persist.MetricSerializer", - "transactionZKServers" : "localhost", - "transactionZKPort" : 2181, - "transactionZKRoot" : "/consumers", - "transactionStateUpdateMS" : 2000 - }, - "alertExecutorConfigs" : { - "hdfsAuditLogAlertExecutor" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner" - "needValidation" : "true" - } - }, - "persistExecutorConfigs" { - "persistExecutor1" : { - "kafka": { - "bootstrap_servers" : "www.xyz.com", - "topics" : { - "defaultOutput" : "downSampleOutput" - } - } - } - }, - "aggregateExecutorConfigs" : { - "aggregateExecutor1" : { - "parallelism" : 1, - "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner" - "needValidation" : "true" - } - }, - "eagleProps" : { - "site" : "sandbox", - "application": "persistTest", - "dataJoinPollIntervalSec" : 30, - "mailHost" : "mailHost.com", - "mailSmtpPort":"25", - "mailDebug" : "true", - "balancePartitionEnabled" : true, - #"partitionRefreshIntervalInMin" : 60, - #"kafkaStatisticRangeInMin" : 60, - "eagleService": { - "host": "localhost", - "port": 9099, - "username": "admin", - "password": "secret" - }, - "readHdfsUserCommandPatternFrom" : "file" - }, - "dynamicConfigSource" : { - "enabled" : true, - "initDelayMillis" : 0, - "delayMillis" : 30000 - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/create-policy-for-plugin-test.sh ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/create-policy-for-plugin-test.sh b/eagle-examples/eagle-topology-example/src/main/resources/create-policy-for-plugin-test.sh deleted file mode 100644 index 3fd2752..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/create-policy-for-plugin-test.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with` -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CUR_DIR=$(dirname $0) -source $CUR_DIR/../../../../../eagle-assembly/src/main/bin/eagle-env.sh - -##################################################################### -# Import stream metadata -##################################################################### - -## AlertDataSource: data sources bound to sites -echo "Importing AlertDataSourceService for persist... " - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService"; \ - -d '[{"prefix":"alertDataSource","tags":{"site" : "sandbox", "dataSource":"testSpout"}, "enabled": "true", "config" : " just some description", "desc":"pluginTest"}]' - - -## AlertStreamService: alert streams generated from data source -echo "" -echo "Importing AlertStreamService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; \ - -d '[{"prefix":"alertStream","tags":{"dataSource":"testSpout","streamName":"testStream"},"desc":"pluginTest"}]' - -## AlertExecutorService: what alert streams are consumed by alert executor -echo "" -echo "Importing AlertExecutorService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; \ - -d '[{"prefix":"alertExecutor","tags":{"dataSource":"testSpout","alertExecutorId":"testExecutor","streamName":"testStream"},"desc":"testStream->testExecutor"}]' - -## AlertStreamSchemaService: schema for event from alert stream -echo "" -echo "Importing AlertStreamSchemaService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ -"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"; \ - -d ' - [ - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "testSpout", - "streamName": "testStream", - "attrName": "testAttribute" - }, - "attrDescription": "testAttribute", - "attrType": "string", - "category": "", - "attrValueResolver": "" - } - ] - ' - -##### add policies ########## -echo "" -echo "Importing policy ... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"; \ - -d ' - [ - { - "prefix": "alertdef", - "tags": { - "site": "sandbox", - "dataSource": "testSpout", - "policyId": "pluginTestPolicy", - "alertExecutorId": "testExecutor", - "policyType": "siddhiCEPEngine" - }, - "description": "pluginTest", - "policyDef": "{\"expression\":\"from testStream[(testAttribute == \\\"testValue\\\")] select * insert into outputStream;\",\"type\":\"siddhiCEPEngine\"}", - "dedupeDef": "{\"alertDedupIntervalMin\":0,\"emailDedupIntervalMin\":1440}", - "notificationDef": "[{\"subject\":\"just for test\",\"sender\":\"[email protected]\",\"recipients\":\"[email protected]\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]", - "remediationDef":"", - "enabled":true - } - ] - ' - -## Finished -echo "" -echo "Finished initialization for NotificationPluginTest" - -exit 0 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/delete-email-for-plugin-test.sh ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/delete-email-for-plugin-test.sh b/eagle-examples/eagle-topology-example/src/main/resources/delete-email-for-plugin-test.sh deleted file mode 100644 index 30d277f..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/delete-email-for-plugin-test.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with` -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CUR_DIR=$(dirname $0) -source $CUR_DIR/../../../../../eagle-assembly/src/main/bin/eagle-env.sh - -##### delete email notification ########## -echo "" -echo "Importing policy ... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"; \ - -d ' - [ - { - "prefix": "alertdef", - "tags": { - "site": "sandbox", - "dataSource": "testSpout", - "policyId": "pluginTestPolicy", - "alertExecutorId": "testExecutor", - "policyType": "siddhiCEPEngine" - }, - "description": "pluginTest", - "policyDef": "{\"expression\":\"from testStream[(testAttribute == \\\"testValue\\\")] select * insert into outputStream;\",\"type\":\"siddhiCEPEngine\"}", - "notificationDef": "[{\"subject\":\"just for test\",\"sender\":\"[email protected]\",\"recipients\":\"[email protected]\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]", - "remediationDef":"", - "enabled":true - } - ] - ' - -## Finished -echo "" -echo "Finished initialization for NotificationPluginTest" - -exit 0 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/log4j.properties b/eagle-examples/eagle-topology-example/src/main/resources/log4j.properties index ff98cd3..d59ded6 100644 --- a/eagle-examples/eagle-topology-example/src/main/resources/log4j.properties +++ b/eagle-examples/eagle-topology-example/src/main/resources/log4j.properties @@ -13,28 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=INFO, stdout, DRFA +log4j.rootLogger=INFO, stdout -eagle.log.dir=./logs -eagle.log.file=eagle.log - - -#log4j.logger.org.apache.eagle.security.auditlog.IPZoneDataJoinBolt=DEBUG -#log4j.logger.org.apache.eagle.security.auditlog.FileSensitivityDataJoinBolt=DEBUG -log4j.logger.org.apache.eagle.security.auditlog.HdfsUserCommandReassembler=DEBUG -#log4j.logger.org.apache.eagle.executor.AlertExecutor=DEBUG # standard output log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n - -# Daily Rolling File Appender -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file} -log4j.appender.DRFA.DatePattern=yyyy-MM-dd -## 30-day backup -# log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/persist-test-topo-init.sh ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/persist-test-topo-init.sh b/eagle-examples/eagle-topology-example/src/main/resources/persist-test-topo-init.sh deleted file mode 100644 index 5805cf3..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/persist-test-topo-init.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with` -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/eagle-env.sh -eagle_bin=$EAGLE_HOME/bin - -##################################################################### -# Import stream metadata for HDFS -##################################################################### - -## AlertDataSource: data sources bound to sites -echo "Importing AlertDataSourceService for persist... " - -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService"; \ - -d '[{"prefix":"alertDataSource","tags":{"site" : "sandbox", "dataSource":"persistTest"}, "enabled": "true", "config" : " just some description", "desc":"persistTest"}]' - - -## AlertStreamService: alert streams generated from data source -echo "" -echo "Importing AlertStreamService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; \ - -d '[{"prefix":"alertStream","tags":{"dataSource":"persistTest","streamName":"persistTestEventStream"},"desc":"persistTest metrics"}]' - -## AlertExecutorService: what alert streams are consumed by alert executor -echo "" -echo "Importing AlertExecutorService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; \ - -d '[{"prefix":"alertExecutor","tags":{"dataSource":"persistTest","alertExecutorId":"aggregateExecutor1","streamName":"persistTestEventStream"},"desc":"aggregate executor for persist test log event stream"}]' - -## AlertStreamSchemaService: schema for event from alert stream -echo "" -echo "Importing AlertStreamSchemaService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ -"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"; \ - -d ' - [ - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "src" - }, - "attrDescription": "source directory or file, such as /tmp", - "attrType": "string", - "category": "", - "attrValueResolver": "org.apache.eagle.service.security.hdfs.resolver.HDFSResourceResolver" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "dst" - }, - "attrDescription": "destination directory, such as /tmp", - "attrType": "string", - "category": "", - "attrValueResolver": "org.apache.eagle.service.security.hdfs.resolver.HDFSResourceResolver" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "host" - }, - "attrDescription": "hostname, such as localhost", - "attrType": "string", - "category": "", - "attrValueResolver": "" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "timestamp" - }, - "attrDescription": "milliseconds of the datetime", - "attrType": "long", - "category": "", - "attrValueResolver": "" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "allowed" - }, - "attrDescription": "true, false or none", - "attrType": "bool", - "category": "", - "attrValueResolver": "" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "user" - }, - "attrDescription": "process user", - "attrType": "string", - "category": "", - "attrValueResolver": "" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "cmd" - }, - "attrDescription": "file/directory operation, such as getfileinfo, open, listStatus and so on", - "attrType": "string", - "category": "", - "attrValueResolver": "org.apache.eagle.service.security.hdfs.resolver.HDFSCommandResolver" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "sensitivityType" - }, - "attrDescription": "mark such as AUDITLOG, SECURITYLOG", - "attrType": "string", - "category": "", - "attrValueResolver": "org.apache.eagle.service.security.hdfs.resolver.HDFSSensitivityTypeResolver" - }, - { - "prefix": "alertStreamSchema", - "tags": { - "dataSource": "persistTest", - "streamName": "persistTestEventStream", - "attrName": "securityZone" - }, - "attrDescription": "", - "attrType": "string", - "category": "", - "attrValueResolver": "" - } - ] - ' - -##### add policies ########## -echo "" -echo "Importing AlertStreamSchemaService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \ - "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AggregateDefinitionService"; \ - -d ' - [ - { - "prefix": "aggregatedef", - "tags": { - "site": "sandbox", - "dataSource": "persistTest", - "policyId": "persitTestPolicy1", - "executorId": "aggregateExecutor1", - "policyType": "siddhiCEPEngine" - }, - "description": "persistetest", - "policyDef": "{\"expression\":\"from persistTestEventStream[(logLevel == 'ERROR')] select * insert into outputStream;\",\"type\":\"siddhiCEPEngine\"}", - "enabled": true - } - ] - ' - - - - ## Finished -echo "" -echo "Finished initialization for eagle topology" - -exit 0 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/eagle-topology-example/src/main/resources/persit-test-storm.yaml ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-topology-example/src/main/resources/persit-test-storm.yaml b/eagle-examples/eagle-topology-example/src/main/resources/persit-test-storm.yaml deleted file mode 100644 index a68a323..0000000 --- a/eagle-examples/eagle-topology-example/src/main/resources/persit-test-storm.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -topology.workers: 1 -topology.acker.executors: 1 -topology.tasks: 1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-examples/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-examples/pom.xml b/eagle-examples/pom.xml index 108c649..273e934 100644 --- a/eagle-examples/pom.xml +++ b/eagle-examples/pom.xml @@ -26,7 +26,6 @@ <artifactId>eagle-examples</artifactId> <packaging>pom</packaging> <modules> - <module>eagle-topology-example</module> <module>eagle-app-example</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-gc/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-gc/pom.xml b/eagle-gc/pom.xml index cb31156..be38567 100644 --- a/eagle-gc/pom.xml +++ b/eagle-gc/pom.xml @@ -43,11 +43,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-alert-process</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-app-base</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-gc/src/main/java/org/apache/eagle/gc/executor/GCLogAnalyzerBolt.java ---------------------------------------------------------------------- diff --git a/eagle-gc/src/main/java/org/apache/eagle/gc/executor/GCLogAnalyzerBolt.java b/eagle-gc/src/main/java/org/apache/eagle/gc/executor/GCLogAnalyzerBolt.java index 59720a3..66cb859 100644 --- a/eagle-gc/src/main/java/org/apache/eagle/gc/executor/GCLogAnalyzerBolt.java +++ b/eagle-gc/src/main/java/org/apache/eagle/gc/executor/GCLogAnalyzerBolt.java @@ -25,19 +25,14 @@ import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; -import com.typesafe.config.Config; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor2; import org.apache.eagle.gc.model.GCPausedEvent; import org.apache.eagle.gc.stream.GCStreamBuilder; import org.apache.eagle.gc.parser.exception.IgnoredLogFormatException; import org.apache.eagle.gc.parser.exception.UnrecognizedLogFormatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; import java.util.Arrays; -import java.util.List; import java.util.Map; public class GCLogAnalyzerBolt extends BaseRichBolt { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-hadoop-metric/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-hadoop-metric/pom.xml b/eagle-hadoop-metric/pom.xml index 389a481..0d612df 100644 --- a/eagle-hadoop-metric/pom.xml +++ b/eagle-hadoop-metric/pom.xml @@ -32,11 +32,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-app-base</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxMetricMonitor.java ---------------------------------------------------------------------- diff --git a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxMetricMonitor.java b/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxMetricMonitor.java deleted file mode 100644 index c0b183b..0000000 --- a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxMetricMonitor.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.hadoop.metric; - -import org.apache.eagle.datastream.ExecutionEnvironments; -import org.apache.eagle.datastream.storm.StormExecutionEnvironment; - -/** - * Created on 1/12/16. - */ -public class HadoopJmxMetricMonitor { - - public static void main(String[] args) { -// StormExecutionEnvironment env = ExecutionEnvironments.get(args, StormExecutionEnvironment.class); -// String streamName = "hadoopJmxMetricEventStream"; -// StreamProducer sp = env.fromSpout(Utils.createProvider(env.getConfig())).withOutputFields(2).nameAs(streamName); -// sp.alertWithConsumer(streamName, "hadoopJmxMetricAlertExecutor"); -// env.execute(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-metric-collection/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-metric-collection/pom.xml b/eagle-security/eagle-metric-collection/pom.xml index 8d7377f..da7587d 100644 --- a/eagle-security/eagle-metric-collection/pom.xml +++ b/eagle-security/eagle-metric-collection/pom.xml @@ -77,11 +77,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-alert-process</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-stream-process-base</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaMessageDistributionBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaMessageDistributionBolt.java b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaMessageDistributionBolt.java index 0caa64c..76c7bf2 100644 --- a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaMessageDistributionBolt.java +++ b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaMessageDistributionBolt.java @@ -28,15 +28,11 @@ import com.codahale.metrics.MetricRegistry; import com.typesafe.config.Config; import org.apache.commons.lang.time.DateUtils; import org.apache.eagle.common.config.EagleConfigConstants; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor1; import org.apache.eagle.metric.reportor.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple1; import java.util.HashMap; -import java.util.List; import java.util.Map; public class KafkaMessageDistributionBolt extends BaseRichBolt { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-common/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/pom.xml b/eagle-security/eagle-security-common/pom.xml index 224869a..bff3897 100644 --- a/eagle-security/eagle-security-common/pom.xml +++ b/eagle-security/eagle-security-common/pom.xml @@ -36,17 +36,6 @@ <artifactId>eagle-stream-process-api</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> - <artifactId>eagle-alert-process</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>org.wso2.orbit.com.lmax</groupId> - <artifactId>disruptor</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.apache.eagle</groupId> <artifactId>eagle-alert-service</artifactId> @@ -54,6 +43,11 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> + <artifactId>eagle-metric</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.eagle</groupId> <artifactId>eagle-metadata-base</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java index a595902..e13a54b 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java @@ -20,39 +20,19 @@ package org.apache.eagle.security.resolver; import com.typesafe.config.*; -import org.apache.eagle.alert.entity.SiteApplicationServiceEntity; -import org.apache.eagle.common.config.EagleConfigConstants; -import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; -import org.apache.eagle.policy.common.Constants; -import org.apache.eagle.service.generic.GenericEntityServiceResource; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; import java.util.Map; +@Deprecated public class MetadataAccessConfigRepo { private static Logger LOG = LoggerFactory.getLogger(MetadataAccessConfigRepo.class); public Config getConfig(String application, String siteId) throws Exception { - - GenericEntityServiceResource resource = new GenericEntityServiceResource(); - String queryFormat = Constants.SITE_APPLICATION_SERVICE_ENDPOINT_NAME + "[@application=\"%s\" AND @site=\"%s\"]{*}"; - GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat, application, siteId), null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); - List<SiteApplicationServiceEntity> list = (List<SiteApplicationServiceEntity>) ret.getObj(); - if (list == null || list.size() == 0) - throw new Exception("Config is empty for site=" + siteId +" application=" + application + "."); - String originalConfigStr = list.get(0).getConfig(); - ConfigParseOptions options = ConfigParseOptions.defaults() - .setSyntax(ConfigSyntax.PROPERTIES) - .setAllowMissing(false); - Config originalConfig = ConfigFactory.parseString(originalConfigStr, options); - if(!originalConfig.hasPath(EagleConfigConstants.CLASSIFICATION_CONFIG)) { - throw new Exception("Fail to get CLASSIFICATION_CONFIG for data classification"); - } - return originalConfig.getConfig(EagleConfigConstants.CLASSIFICATION_CONFIG); + return null; } public Configuration convert(Config originalConfig) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hbase-auditlog/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/pom.xml b/eagle-security/eagle-security-hbase-auditlog/pom.xml index 46d67b5..009f869 100644 --- a/eagle-security/eagle-security-hbase-auditlog/pom.xml +++ b/eagle-security/eagle-security-hbase-auditlog/pom.xml @@ -36,11 +36,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-app-base</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hdfs-auditlog/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/pom.xml b/eagle-security/eagle-security-hdfs-auditlog/pom.xml index 18c4908..f137ad6 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/pom.xml +++ b/eagle-security/eagle-security-hdfs-auditlog/pom.xml @@ -33,11 +33,11 @@ <artifactId>eagle-security-common</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-policy-base</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.eagle</groupId> <artifactId>eagle-embed-server</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java index 6cbbde6..f2582f1 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java @@ -18,14 +18,11 @@ package org.apache.eagle.security.auditlog; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; -import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import com.typesafe.config.Config; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor2; import org.apache.eagle.security.auditlog.timer.FileSensitivityPollingJob; import org.apache.eagle.security.auditlog.util.SimplifyPath; import org.apache.eagle.security.entity.FileSensitivityAPIEntity; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java index 1b2df3b..2ce0dc1 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java @@ -18,11 +18,9 @@ */ package org.apache.eagle.security.auditlog; +import backtype.storm.task.OutputCollector; import com.typesafe.config.Config; import org.apache.eagle.policy.siddhi.AttributeType; -import org.apache.eagle.policy.siddhi.SiddhiStreamMetadataUtils; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor2; import org.apache.eagle.security.entity.HdfsUserCommandPatternEntity; import org.apache.eagle.service.client.EagleServiceConnector; import org.slf4j.Logger; @@ -32,11 +30,10 @@ import org.wso2.siddhi.core.SiddhiManager; import org.wso2.siddhi.core.event.Event; import org.wso2.siddhi.core.query.output.callback.QueryCallback; import org.wso2.siddhi.core.stream.input.InputHandler; -import scala.Tuple2; import java.util.*; -public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, Map> { +public class HdfsUserCommandReassembler { private static final Logger LOG = LoggerFactory.getLogger(HdfsUserCommandReassembler.class); private Config config; private InputHandler inputHandler; @@ -54,7 +51,6 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, put("cmd", AttributeType.STRING.name()); }}; - @Override public void prepareConfig(Config config) { this.config = config; } @@ -69,7 +65,7 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, @Override public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) { Object[] attrValues = inEvents[0].getData(); - Collector<Tuple2<String, Map>> collector = (Collector<Tuple2<String, Map>>)attrValues[0]; + OutputCollector collector = (OutputCollector) attrValues[0]; SortedMap<String, Object> outputEvent = new TreeMap<String, Object>(); int i = 1; // output is from second element String user = null; @@ -82,13 +78,47 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, outputEvent.putAll(outputModifier); LOG.debug("outputEvent: " + outputEvent); - collector.collect(new Tuple2<String, Map>(user, outputEvent)); + collector.emit(Arrays.asList(user, outputEvent)); } } - @Override + public String convertToStreamDef(String streamName, Map<String, String> eventSchema){ + StringBuilder sb = new StringBuilder(); + sb.append("context" + " object,"); + for(Map.Entry<String, String> entry : eventSchema.entrySet()){ + appendAttributeNameType(sb, entry.getKey(), entry.getValue()); + } + if(sb.length() > 0){ + sb.deleteCharAt(sb.length()-1); + } + + String siddhiStreamDefFormat = "define stream " + streamName + "(" + "%s" + ");"; + return String.format(siddhiStreamDefFormat, sb.toString()); + } + + private void appendAttributeNameType(StringBuilder sb, String attrName, String attrType){ + sb.append(attrName); + sb.append(" "); + if(attrType.equalsIgnoreCase(AttributeType.STRING.name())){ + sb.append("string"); + }else if(attrType.equalsIgnoreCase(AttributeType.INTEGER.name())){ + sb.append("int"); + }else if(attrType.equalsIgnoreCase(AttributeType.LONG.name())){ + sb.append("long"); + }else if(attrType.equalsIgnoreCase(AttributeType.BOOL.name())){ + sb.append("bool"); + }else if(attrType.equalsIgnoreCase(AttributeType.FLOAT.name())){ + sb.append("float"); + }else if(attrType.equalsIgnoreCase(AttributeType.DOUBLE.name())){ + sb.append("double"); + }else{ + LOG.warn("AttrType is not recognized, ignore : " + attrType); + } + sb.append(","); + } + public void init() { - String streamDef = SiddhiStreamMetadataUtils.convertToStreamDef(streamName, eventSchema); + String streamDef = convertToStreamDef(streamName, eventSchema); SiddhiManager siddhiManager = new SiddhiManager(); StringBuilder sb = new StringBuilder(); sb.append(streamDef); @@ -137,8 +167,7 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, executionPlanRuntime.start(); } - @Override - public void flatMap(List<Object> input, Collector<Tuple2<String, Map>> collector) { + public void flatMap(List<Object> input, OutputCollector collector) { if(LOG.isDebugEnabled()) LOG.debug("incoming event:" + input.get(1)); SortedMap<String, Object> toBeCopied = (SortedMap<String, Object>) input.get(1); SortedMap<String, Object> event = new TreeMap<>(toBeCopied); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/IPZoneDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/IPZoneDataJoinBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/IPZoneDataJoinBolt.java index d02f959..790086d 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/IPZoneDataJoinBolt.java +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/IPZoneDataJoinBolt.java @@ -23,16 +23,12 @@ import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import com.typesafe.config.Config; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor2; -import org.apache.eagle.security.auditlog.timer.FileSensitivityPollingJob; import org.apache.eagle.security.auditlog.timer.IPZonePollingJob; import org.apache.eagle.security.entity.IPZoneEntity; import org.apache.eagle.security.util.ExternalDataCache; import org.apache.eagle.security.util.ExternalDataJoiner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; import java.util.Arrays; import java.util.Map; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hdfs-auditlog/src/test/java/org/apache/eagle/security/auditlog/TestUserCommandReassembler.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/test/java/org/apache/eagle/security/auditlog/TestUserCommandReassembler.java b/eagle-security/eagle-security-hdfs-auditlog/src/test/java/org/apache/eagle/security/auditlog/TestUserCommandReassembler.java index 753eb41..caea37a 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/test/java/org/apache/eagle/security/auditlog/TestUserCommandReassembler.java +++ b/eagle-security/eagle-security-hdfs-auditlog/src/test/java/org/apache/eagle/security/auditlog/TestUserCommandReassembler.java @@ -19,14 +19,14 @@ package org.apache.eagle.security.auditlog; +import backtype.storm.task.IOutputCollector; +import backtype.storm.task.OutputCollector; +import backtype.storm.tuple.Tuple; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; -import org.apache.eagle.datastream.Collector; import org.apache.eagle.security.hdfs.HDFSAuditLogObject; import org.apache.eagle.security.hdfs.HDFSAuditLogParser; -import org.junit.Assert; import org.junit.Test; -import scala.Tuple2; import java.util.*; @@ -62,13 +62,32 @@ public class TestUserCommandReassembler { assembler.prepareConfig(config); assembler.init(); - Collector<Tuple2<String, Map>> collector = new Collector<Tuple2<String, Map>>(){ + OutputCollector collector = new OutputCollector(new IOutputCollector() { @Override - public void collect(Tuple2<String, Map> stringMapTuple2) { - String cmd = (String)stringMapTuple2._2().get("cmd");Assert.assertEquals("user:appendToFile", cmd); - System.out.println("assert passed!!!"); + public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) { + return null; } - }; + + @Override + public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) { + + } + + @Override + public void ack(Tuple input) { + + } + + @Override + public void fail(Tuple input) { + + } + + @Override + public void reportError(Throwable error) { + + } + }); assembler.flatMap(Arrays.asList("user1", parseEvent(e1)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e2)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e3)), collector); @@ -91,14 +110,32 @@ public class TestUserCommandReassembler { assembler.prepareConfig(config); assembler.init(); - Collector<Tuple2<String, Map>> collector = new Collector<Tuple2<String, Map>>(){ + OutputCollector collector = new OutputCollector(new IOutputCollector() { + @Override + public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) { + return null; + } + + @Override + public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) { + + } + + @Override + public void ack(Tuple input) { + + } + + @Override + public void fail(Tuple input) { + + } + @Override - public void collect(Tuple2<String, Map> stringMapTuple2) { - String cmd = (String)stringMapTuple2._2().get("cmd"); - Assert.assertEquals("user:read", cmd); - System.out.println("assert passed!!!"); + public void reportError(Throwable error) { + } - }; + }); assembler.flatMap(Arrays.asList("user1", parseEvent(e1)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e3)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e2)), collector); @@ -128,15 +165,32 @@ public class TestUserCommandReassembler { assembler.prepareConfig(config); assembler.init(); - Collector<Tuple2<String, Map>> collector = new Collector<Tuple2<String, Map>>(){ + OutputCollector collector = new OutputCollector(new IOutputCollector() { + @Override + public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) { + return null; + } + + @Override + public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) { + + } + + @Override + public void ack(Tuple input) { + + } + + @Override + public void fail(Tuple input) { + + } + @Override - public void collect(Tuple2<String, Map> stringMapTuple2) { - String cmd = (String)stringMapTuple2._2().get("cmd"); - Assert.assertEquals("user:copyFromLocal", cmd); - //Assert.assertEquals("user:appendToFile", cmd); - System.out.println("assert passed!!!"); + public void reportError(Throwable error) { + } - }; + }); assembler.flatMap(Arrays.asList("user1", parseEvent(e1)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e2)), collector); assembler.flatMap(Arrays.asList("user1", parseEvent(e3)), collector); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hive/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hive/pom.xml b/eagle-security/eagle-security-hive/pom.xml index 1d4bdc8..d431d46 100644 --- a/eagle-security/eagle-security-hive/pom.xml +++ b/eagle-security/eagle-security-hive/pom.xml @@ -34,11 +34,6 @@ <artifactId>eagle-storm-jobrunning-spout</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-framework</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/sensitivity/HiveResourceSensitivityDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/sensitivity/HiveResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/sensitivity/HiveResourceSensitivityDataJoinBolt.java index 57da65b..b7c1765 100644 --- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/sensitivity/HiveResourceSensitivityDataJoinBolt.java +++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/sensitivity/HiveResourceSensitivityDataJoinBolt.java @@ -23,15 +23,11 @@ import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import com.typesafe.config.Config; -import org.apache.eagle.datastream.Collector; -import org.apache.eagle.datastream.JavaStormStreamExecutor2; -import org.apache.eagle.jobrunning.storm.JobRunningContentFilterImpl; import org.apache.eagle.security.entity.HiveResourceSensitivityAPIEntity; import org.apache.eagle.security.util.ExternalDataCache; import org.apache.eagle.security.util.ExternalDataJoiner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; import java.util.Arrays; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-maprfs-auditlog/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-maprfs-auditlog/pom.xml b/eagle-security/eagle-security-maprfs-auditlog/pom.xml index 9ba0ac1..174d4d0 100644 --- a/eagle-security/eagle-security-maprfs-auditlog/pom.xml +++ b/eagle-security/eagle-security-maprfs-auditlog/pom.xml @@ -35,11 +35,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-security-hdfs-auditlog</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/log4j.properties b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/log4j.properties index 12f215c..d59ded6 100644 --- a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/log4j.properties +++ b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/log4j.properties @@ -13,28 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=INFO, stdout, DRFA +log4j.rootLogger=INFO, stdout -eagle.log.dir=./logs -eagle.log.file=eagle.log - - -#log4j.logger.org.apache.eagle.security.auditlog.IPZoneDataJoinExecutor=DEBUG -#log4j.logger.org.apache.eagle.security.auditlog.FileSensitivityDataJoinBoltUG -log4j.logger.org.apache.eagle.security.auditlog.HdfsUserCommandReassembler=DEBUG -#log4j.logger.org.apache.eagle.executor.AlertExecutor=DEBUG # standard output log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n - -# Daily Rolling File Appender -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file} -log4j.appender.DRFA.DatePattern=yyyy-MM-dd -## 30-day backup -# log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-server-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-server-assembly/pom.xml b/eagle-server-assembly/pom.xml index 42d2b04..803f36f 100644 --- a/eagle-server-assembly/pom.xml +++ b/eagle-server-assembly/pom.xml @@ -24,7 +24,7 @@ <artifactId>eagle-parent</artifactId> <version>0.5.0-incubating-SNAPSHOT</version> </parent> - <groupId>eagle</groupId> + <groupId>org.apache.eagle</groupId> <artifactId>eagle-server-assembly</artifactId> <name>eagle-server-assembly</name> <url>http://maven.apache.org</url> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0f11a591/eagle-webservice/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-webservice/pom.xml b/eagle-webservice/pom.xml index 9b00f66..ec2c913 100644 --- a/eagle-webservice/pom.xml +++ b/eagle-webservice/pom.xml @@ -190,12 +190,6 @@ <artifactId>eagle-security-maprfs-web</artifactId> <version>${project.version}</version> </dependency> - <!-- - <dependency> - <groupId>org.apache.eagle</groupId> - <artifactId>eagle-stream-application-manager</artifactId> - <version>${project.version}</version> - </dependency> --> <dependency> <groupId>org.apache.eagle</groupId> <artifactId>eagle-topology-assembly</artifactId> @@ -216,17 +210,6 @@ </dependency> <dependency> <groupId>org.apache.eagle</groupId> - <artifactId>eagle-alert-base</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.storm</groupId> - <artifactId>storm-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.eagle</groupId> <artifactId>eagle-alert-service</artifactId> <version>${project.version}</version> </dependency>
