Merge branch 'branch-0.4.0'
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a0443561 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a0443561 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a0443561 Branch: refs/heads/master Commit: a04435619023843cb33fda74092b9830df5141bc Parents: 982a6b7 eac0f27 Author: anyway1021 <m...@apache.org> Authored: Mon Jul 25 17:21:31 2016 +0800 Committer: anyway1021 <m...@apache.org> Committed: Mon Jul 25 17:21:31 2016 +0800 ---------------------------------------------------------------------- CHANGELOG.txt | 327 +++++++++++++ README.md | 10 +- eagle-assembly/pom.xml | 4 +- eagle-assembly/src/assembly/eagle-bin.xml | 7 +- .../src/main/bin/eagle-create-table.rb | 47 -- .../src/main/bin/eagle-drop-tables.sh | 26 - eagle-assembly/src/main/bin/eagle-env.sh | 4 - .../src/main/bin/eagle-service-init.sh | 35 -- eagle-assembly/src/main/bin/eagle-service.sh | 7 + .../src/main/bin/eagle-topology-init.sh | 33 +- .../src/main/conf/eagle-scheduler.conf | 42 ++ eagle-assembly/src/main/conf/eagle-service.conf | 4 +- .../sandbox-hbaseSecurityLog-application.conf | 2 +- .../src/main/docs/logstash-kafka-conf.md | 76 ++- .../src/main/examples/eagle-sandbox-starter.sh | 61 +-- .../sample-sensitivity-resource-create.sh | 2 + .../src/main/lib/tomcat/bin/bootstrap.jar | Bin 28052 -> 0 bytes .../src/main/lib/tomcat/bin/commons-daemon.jar | Bin 24283 -> 0 bytes .../src/main/lib/tomcat/bin/tomcat-juli.jar | Bin 38222 -> 0 bytes eagle-core/eagle-alert/eagle-alert-base/pom.xml | 2 +- .../eagle-alert-notification-plugin/pom.xml | 8 +- .../notification/email/AlertEmailGenerator.java | 31 +- .../notification/email/AlertEmailSender.java | 60 ++- .../src/main/resources/ALERT_DEFAULT.vm | 488 ++++++++++--------- .../eagle-alert/eagle-alert-process/pom.xml | 2 +- .../eagle/alert/config/DeduplicatorConfig.java | 17 +- .../dedup/AlertDeduplicationExecutorBase.java | 22 +- .../eagle/alert/dedup/DefaultDeduplicator.java | 50 +- .../eagle/alert/dedup/EntityDedupKey.java | 62 +++ .../eagle/alert/dedup/EntityDeduplicator.java | 9 +- .../executor/AlertExecutorCreationUtils.java | 2 +- .../siddhi/SiddhiAlertAPIEntityRender.java | 18 +- .../eagle/alert/config/TestAlertDedup.java | 6 +- .../TestSiddhiStateSnapshotAndRestore.java | 7 +- .../eagle-alert/eagle-alert-service/pom.xml | 2 +- .../service/alert/SiteApplicationObject.java | 62 +++ .../service/alert/SiteApplicationResource.java | 189 +++++++ .../alert/resolver/SiteApplicationObject.java | 62 --- .../alert/resolver/SiteApplicationResource.java | 189 ------- eagle-core/eagle-alert/pom.xml | 2 +- .../eagle-application-service/pom.xml | 56 +++ .../application/AppManagerConstants.java | 43 ++ .../ApplicationManagementResource.java | 109 +++++ .../application/dao/ApplicationManagerDAO.java | 33 ++ .../dao/ApplicationManagerDaoImpl.java | 91 ++++ .../entity/ApplicationEntityRepo.java | 30 ++ .../entity/TopologyDescriptionEntity.java | 104 ++++ .../entity/TopologyExecutionEntity.java | 132 +++++ .../entity/TopologyExecutionStatus.java | 38 ++ .../entity/TopologyOperationEntity.java | 105 ++++ .../eagle-stream-application-manager/pom.xml | 144 ++++++ .../stream/application/TopologyException.java | 26 + .../stream/application/TopologyExecutable.java | 27 + .../stream/application/TopologyFactory.java | 55 +++ .../AbstractDynamicApplication.scala | 32 ++ .../stream/application/ApplicationManager.scala | 126 +++++ .../application/ApplicationManagerUtils.scala | 38 ++ .../ApplicationSchedulerAsyncDAO.scala | 179 +++++++ .../stream/application/ExecutionPlatform.scala | 30 ++ .../application/ExecutionPlatformFactory.scala | 49 ++ .../eagle/stream/application/TaskExecutor.scala | 41 ++ .../application/impl/StormDynamicTopology.scala | 44 ++ .../impl/StormExecutionPlatform.scala | 197 ++++++++ .../scheduler/AppCommandExecutor.scala | 170 +++++++ .../scheduler/AppCommandLoader.scala | 78 +++ .../scheduler/ApplicationScheduler.scala | 81 +++ .../scheduler/StreamAppCoordinator.scala | 54 ++ .../src/test/resources/application.conf | 42 ++ .../src/test/resources/log4j.properties | 35 ++ .../application/scheduler/MockTopology.scala | 30 ++ .../scheduler/StormApplicationManagerSpec.scala | 40 ++ .../application/scheduler/TestScheduler.scala | 61 +++ eagle-core/eagle-application-management/pom.xml | 40 ++ .../eagle-data-process/eagle-job-common/pom.xml | 2 +- .../eagle-storm-jobrunning-spout/pom.xml | 2 +- .../job/conf/TestJobConfParserImpl.java | 43 ++ .../src/test/resources/jobconf.html | 230 +++++++++ .../eagle-stream-pipeline/pom.xml | 2 +- .../eagle/stream/pipeline/parser/Pipeline.scala | 6 + .../eagle-stream-process-api/pom.xml | 2 +- .../src/main/resources/application.conf | 78 +++ .../src/main/resources/log4j.properties | 40 ++ .../core/StreamParallelismConfigExpansion.scala | 4 +- .../storm/StormTopologyExecutorImpl.scala | 14 +- .../eagle-stream-process-base/pom.xml | 4 +- eagle-core/eagle-data-process/pom.xml | 4 +- .../eagle-embed/eagle-embed-hbase/pom.xml | 2 +- .../eagle-embed/eagle-embed-server/pom.xml | 2 +- eagle-core/eagle-embed/pom.xml | 4 +- .../eagle-machinelearning-base/pom.xml | 4 +- eagle-core/eagle-machinelearning/pom.xml | 4 +- eagle-core/eagle-metric/pom.xml | 4 +- .../eagle-policy/eagle-policy-base/pom.xml | 2 +- .../apache/eagle/policy/common/Constants.java | 5 + .../apache/eagle/policy/common/UrlBuilder.java | 4 +- .../policy/siddhi/SiddhiPolicyEvaluator.java | 57 ++- eagle-core/eagle-policy/pom.xml | 4 +- eagle-core/eagle-query/eagle-antlr/pom.xml | 4 +- eagle-core/eagle-query/eagle-audit-base/pom.xml | 4 +- .../eagle-query/eagle-client-base/pom.xml | 2 +- eagle-core/eagle-query/eagle-common/pom.xml | 2 +- .../org/apache/eagle/common/DateTimeUtil.java | 12 + .../apache/eagle/common/config/EagleConfig.java | 3 + .../common/config/EagleConfigConstants.java | 3 + .../eagle/common/config/EagleConfigFactory.java | 5 + .../eagle/common/email/EagleMailClient.java | 21 +- .../eagle-query/eagle-entity-base/pom.xml | 2 +- eagle-core/eagle-query/eagle-query-base/pom.xml | 4 +- .../eagle-query/eagle-service-base/pom.xml | 4 +- .../eagle-query/eagle-storage-base/pom.xml | 4 +- .../eagle/storage/operation/CompiledQuery.java | 21 +- .../eagle-query/eagle-storage-hbase/pom.xml | 4 +- .../storage/hbase/HBaseEntitySchemaManager.java | 102 ++++ .../eagle/storage/hbase/HBaseStorage.java | 1 + .../eagle-query/eagle-storage-jdbc/pom.xml | 2 +- .../eagle/storage/jdbc/JdbcConstants.java | 7 +- .../criteria/impl/QueryCriteriaBuilder.java | 2 +- .../jdbc/entity/impl/JdbcEntityWriterImpl.java | 21 +- .../schema/JdbcEntityDefinitionManager.java | 2 +- .../jdbc/schema/JdbcEntitySchemaManager.java | 6 +- eagle-core/eagle-query/pom.xml | 2 +- eagle-core/pom.xml | 5 +- eagle-dev/eclipse-java-formatter.xml | 311 ++++++++++++ eagle-docs/images/appManager/admin-page.png | Bin 0 -> 208534 bytes .../images/appManager/start-topology-1.png | Bin 0 -> 212648 bytes .../images/appManager/start-topology-2.png | Bin 0 -> 235826 bytes .../images/appManager/stop-topology-1.png | Bin 0 -> 238503 bytes .../images/appManager/stop-topology-2.png | Bin 0 -> 238064 bytes .../images/appManager/stop-topology-3.png | Bin 0 -> 236337 bytes .../appManager/topology-configuration-1.png | Bin 0 -> 241252 bytes .../appManager/topology-configuration-2.png | Bin 0 -> 288253 bytes .../appManager/topology-configuration-save.png | Bin 0 -> 243119 bytes .../images/appManager/topology-description.png | Bin 0 -> 218471 bytes .../images/appManager/topology-execution.png | Bin 0 -> 219798 bytes .../images/appManager/topology-monitor.png | Bin 0 -> 197752 bytes eagle-docs/images/asf_logo.svg | 16 + eagle-docs/images/notificationPlugin.png | Bin 0 -> 224458 bytes ...erequisites_for_maprFSAuditLog_monitoring.md | 122 +++++ .../alert_notification_plugin_tutorial.md | 87 ++++ .../tutorial/application_manager_tutorial.md | 117 +++++ .../tutorial/getting_started_with_eagle.md | 2 - eagle-examples/eagle-topology-example/pom.xml | 4 +- eagle-examples/pom.xml | 2 +- eagle-external/eagle-kafka/pom.xml | 4 +- eagle-external/eagle-log4jkafka/pom.xml | 4 +- .../hadoop_jmx_collector/metric_collector.py | 4 +- eagle-external/pom.xml | 4 +- eagle-gc/pom.xml | 2 +- eagle-hadoop-metric/pom.xml | 7 +- .../HadoopJmxMetricMonitoringTopology.java | 37 ++ .../src/main/resources/hadoop-metric-init.sh | 2 +- .../src/main/resources/log4j.properties | 5 - eagle-security/eagle-metric-collection/pom.xml | 2 +- eagle-security/eagle-security-common/pom.xml | 2 +- .../security/entity/OozieResourceEntity.java | 82 ++++ .../OozieResourceSensitivityAPIEntity.java | 46 ++ .../entity/SecurityEntityRepository.java | 1 + .../security/hdfs/MAPRFSAuditLogObject.java | 31 ++ .../security/hdfs/MAPRFSAuditLogParser.java | 67 +++ .../resolver/MetadataAccessConfigRepo.java | 15 +- .../AbstractResourceSensitivityPollingJob.java | 2 +- .../eagle/security/util/ExternalDataJoiner.java | 8 +- .../crawler/audit/TestMAPRFSAuditLogParser.java | 64 +++ .../audit/TestMetaDataAccessConfigRepo.java | 50 +- .../eagle-security-hbase-securitylog/pom.xml | 9 +- .../hbase/HbaseAuditLogMonitoringTopology.java | 42 ++ .../parse/HbaseAuditLogKafkaDeserializer.java | 29 +- .../hbase/parse/HbaseAuditLogParser.java | 144 ++---- .../src/main/resources/log4j.properties | 21 + .../hbase/TestHbaseAuditLogProcessTopology.java | 44 ++ .../src/test/resources/application.conf | 71 +++ .../src/test/resources/log4j.properties | 35 ++ eagle-security/eagle-security-hbase-web/pom.xml | 4 +- .../eagle-security-hdfs-auditlog/pom.xml | 7 +- .../HdfsAuditLogMonitoringTopology.java | 40 ++ .../auditlog/HdfsAuditLogProcessorMain.java | 12 +- .../timer/FileSensitivityPollingJob.java | 2 +- .../auditlog/timer/IPZonePollingJob.java | 2 +- .../eagle-security-hdfs-securitylog/pom.xml | 4 +- eagle-security/eagle-security-hdfs-web/pom.xml | 2 +- eagle-security/eagle-security-hive-web/pom.xml | 2 +- .../dao/HiveSensitivityMetadataDAOImpl.java | 2 - .../hive/resolver/HiveMetadataResolver.java | 2 - eagle-security/eagle-security-hive/pom.xml | 9 +- .../hive/HiveJobRunningMonitoringTopology.java | 48 ++ ...HiveJobRunningSourcedStormSpoutProvider.java | 9 +- .../apache/eagle/security/hive/ql/Parser.java | 52 +- .../HiveResourceSensitivityPollingJob.java | 2 +- .../eagle/security/hive/ql/TestParser.java | 64 ++- .../eagle-security-maprfs-auditlog/pom.xml | 74 +++ .../MapRFSAuditLogKafkaDeserializer.java | 69 +++ .../auditlog/MapRFSAuditLogProcessorMain.java | 115 +++++ .../src/main/resources/application.conf | 69 +++ .../src/main/resources/log4j.properties | 40 ++ .../src/main/resources/maprFSAuditLog-init.sh | 213 ++++++++ .../main/resources/security-auditlog-storm.yaml | 18 + .../eagle-security-maprfs-web/pom.xml | 70 +++ .../security/hdfs/MAPRFSResourceConstants.java | 24 + .../hdfs/resolver/MAPRFSCommandResolver.java | 72 +++ .../hdfs/resolver/MAPRStatusCodeResolver.java | 81 +++ .../hdfs/rest/MAPRFSResourceWebResource.java | 73 +++ .../eagle-security-oozie-auditlog/pom.xml | 40 ++ .../parse/OozieAuditLogKafkaDeserializer.java | 71 +++ .../oozie/parse/OozieAuditLogObject.java | 35 ++ .../oozie/parse/OozieAuditLogParser.java | 110 +++++ .../oozie/parse/OozieAuditLogProcessorMain.java | 33 ++ ...ozieResourceSensitivityDataJoinExecutor.java | 89 ++++ .../OozieResourceSensitivityPollingJob.java | 63 +++ .../src/main/resources/application.conf | 66 +++ .../oozie/parse/TestOozieAuditLogParser.java | 56 +++ eagle-security/eagle-security-oozie-web/pom.xml | 91 ++++ .../OozieResourceSensitivityDataJoiner.java | 47 ++ .../BadOozieMetadataAccessConfigException.java | 27 + .../oozie/dao/OozieMetadataAccessConfig.java | 87 ++++ .../oozie/dao/OozieMetadataAccessConfigDAO.java | 23 + .../dao/OozieMetadataAccessConfigDAOImpl.java | 29 ++ .../security/oozie/dao/OozieMetadataDAO.java | 25 + .../oozie/dao/OozieMetadataDAOImpl.java | 54 ++ .../oozie/dao/OozieSensitivityMetadataDAO.java | 29 ++ .../dao/OozieSensitivityMetadataDAOImpl.java | 81 +++ .../res/OozieMetadataBrowseWebResource.java | 58 +++ .../res/OozieMetadataBrowseWebResponse.java | 44 ++ .../TestOozieResourceSensitivityDataJoiner.java | 92 ++++ .../dao/TestOozieMetadataAccessConfig.java | 43 ++ .../TestOozieSensitivityMetadataDAOImpl.java | 120 +++++ .../src/test/resources/coordinatorJob.json | 94 ++++ .../eagle-security-userprofile/common/pom.xml | 2 +- .../detection/pom.xml | 2 +- .../eagle-security-userprofile/pom.xml | 4 +- .../eagle-security-userprofile/training/pom.xml | 4 +- eagle-security/pom.xml | 53 +- eagle-topology-assembly/pom.xml | 46 +- .../src/assembly/eagle-topology-assembly.xml | 4 +- eagle-webservice/pom.xml | 77 ++- .../profile/ApplicationSchedulerListener.java | 63 +++ .../profile/EagleServiceProfileInitializer.java | 3 + .../src/main/resources/application-derby.conf | 30 ++ .../src/main/resources/eagle-scheduler.conf | 42 ++ .../src/main/resources/log4j.properties | 7 +- eagle-webservice/src/main/webapp/README.md | 19 + .../src/main/webapp/WEB-INF/web.xml | 6 + .../src/main/webapp/app/public/css/main.css | 8 + .../public/feature/classification/controller.js | 58 ++- .../classification/page/sensitivity/job.html | 92 ++++ .../app/public/feature/common/controller.js | 24 +- .../app/public/feature/metrics/controller.js | 2 +- .../app/public/feature/topology/controller.js | 257 ++++++++++ .../feature/topology/page/management.html | 52 ++ .../feature/topology/page/monitoring.html | 151 ++++++ .../src/main/webapp/app/public/js/app.config.js | 15 +- .../src/main/webapp/app/public/js/app.js | 13 +- .../src/main/webapp/app/public/js/common.js | 49 +- .../public/js/ctrl/configurationController.js | 10 +- .../webapp/app/public/js/srv/entitiesSrv.js | 30 +- .../main/webapp/app/public/js/srv/siteSrv.js | 2 +- .../src/main/webapp/app/public/js/srv/uiSrv.js | 24 +- mkdocs.yml | 2 + pom.xml | 324 ++++++++---- 258 files changed, 9500 insertions(+), 1263 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a0443561/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java ----------------------------------------------------------------------