EAGLE-190 JBDC metadata storage extension and support derby embedded database as default storage
https://issues.apache.org/jira/browse/EAGLE-190 * Support embedded Derby in local/memory: The JDBC can support derby and mysql completely, and use derby in memory by default, so that user donât have to install any additional dependencies even mysql to get started, and also our unit test could easily pass without any tricky modifications. * Automatic DDL Schema Manager: JdbcEntitySchemaManager could automatically create all required tables if not existing when application is started. * Fixed JDBC reserved field name like desc, group and so on * Force update while inserting failed due to duplicated key exception Closes #118. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ffd84fa0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ffd84fa0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ffd84fa0 Branch: refs/heads/master Commit: ffd84fa0e2dcf2fd3a8d444e1b589341789f6bcd Parents: 4c1831a Author: Hao Chen <h...@apache.org> Authored: Wed Mar 16 13:17:10 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Mar 16 13:17:10 2016 +0800 ---------------------------------------------------------------------- .../src/main/bin/eagle-topology-init.sh | 32 +- eagle-assembly/src/main/conf/eagle-service.conf | 37 +-- .../src/main/lib/jdbc/eagle-jdbc-mysql.sql | 331 +++++++++++++++++++ .../notification/email/AlertEmailGenerator.java | 8 +- .../testcases/TestAlertKafkaPlugin.java | 6 +- .../alert/notification/AlertEmailGenerator.java | 135 ++++++++ .../siddhi/SiddhiAlertAPIEntityRender.java | 2 +- .../TestSiddhiStateSnapshotAndRestore.java | 1 + .../eagle/ml/impl/MLAnomalyCallbackImpl.java | 2 +- .../eagle/alert/entity/AlertAPIEntity.java | 23 +- .../alert/entity/AlertDataSourceEntity.java | 10 +- .../alert/entity/AlertDefinitionAPIEntity.java | 13 +- .../eagle/alert/entity/AlertExecutorEntity.java | 14 +- .../eagle/alert/entity/AlertStreamEntity.java | 16 +- .../entity/ApplicationDescServiceEntity.java | 24 +- .../alert/entity/FeatureDescServiceEntity.java | 12 +- .../org/apache/eagle/common/DateTimeUtil.java | 2 +- .../eagle/common/config/EagleConfigFactory.java | 4 +- .../eagle/common/metric/AlertContext.java | 33 +- .../eagle/common/metric/TestAlertContext.java | 39 +++ .../eagle/log/entity/meta/EntityDefinition.java | 8 + .../entity/meta/EntityDefinitionManager.java | 12 +- .../entity/test/TestTimeSeriesAPIEntity.java | 8 +- .../eagle/storage/DataStorageManager.java | 4 +- .../eagle/storage/operation/CompiledQuery.java | 5 + .../eagle-query/eagle-storage-jdbc/pom.xml | 4 + .../eagle/storage/jdbc/JdbcConstants.java | 9 + .../apache/eagle/storage/jdbc/JdbcStorage.java | 25 +- .../impl/EncodedRowkeyPrimaryKeyBuilder.java | 45 +++ .../jdbc/conn/impl/TorqueStatementPeerImpl.java | 3 +- .../impl/ExpressionCriterionBuilder.java | 64 +++- .../criteria/impl/QueryCriteriaBuilder.java | 27 +- .../jdbc/entity/JdbcEntitySerDeserHelper.java | 72 +++- .../storage/jdbc/entity/JdbcEntityUtils.java | 26 ++ .../storage/jdbc/entity/JdbcEntityWriter.java | 2 +- .../jdbc/entity/impl/JdbcEntityDeleterImpl.java | 2 +- .../jdbc/entity/impl/JdbcEntityReaderImpl.java | 5 +- .../jdbc/entity/impl/JdbcEntityUpdaterImpl.java | 10 +- .../jdbc/entity/impl/JdbcEntityWriterImpl.java | 27 +- .../jdbc/schema/IJdbcEntityDDLManager.java | 26 ++ .../jdbc/schema/JdbcEntityDefinition.java | 82 +++-- .../schema/JdbcEntityDefinitionManager.java | 54 ++- .../jdbc/schema/JdbcEntitySchemaManager.java | 224 +++++++++++++ .../jdbc/schema/ddl/JdbcEntityDdlManager.java | 24 -- .../storage/jdbc/schema/ddl/package-info.java | 24 -- .../eagle/storage/jdbc/schema/package-info.java | 24 ++ .../schema/serializer/DefaultJdbcSerDeser.java | 29 +- .../jdbc/schema/serializer/JdbcSerDeser.java | 20 +- .../schema/serializer/JsonJdbcSerDeser.java | 10 +- .../eagle/storage/jdbc/TestJdbcStorage.java | 79 +++-- .../jdbc/conn/TestConnectionFactory.java | 55 --- .../jdbc/criteria/TestQueryCriteriaBuilder.java | 13 +- .../jdbc/schema/TestJdbcEntityDefinition.java | 33 ++ .../schema/TestJdbcEntitySchemaManager.java | 27 ++ .../src/test/resources/application-derby.conf | 28 ++ .../src/test/resources/application-mysql.conf | 28 ++ .../src/test/resources/application.conf | 6 +- .../resources/ddl/unittest_testtsentity.sql | 43 --- .../src/test/resources/log4j.properties | 2 +- .../test/resources/unittest.application.conf | 28 ++ .../src/test/resources/unittest_ddl.derby.sql | 34 ++ .../src/test/resources/unittest_ddl.mysql.sql | 44 +++ .../src/main/resources/hadoop-metric-init.sh | 24 +- .../hbase/HbaseSensitivityResourceService.java | 16 +- .../hdfs/HDFSResourceSensitivityService.java | 16 +- .../dao/HiveMetadataAccessConfigDAOImpl.java | 45 ++- .../dao/HiveSensitivityMetadataDAOImpl.java | 14 +- eagle-webservice/pom.xml | 6 + .../src/main/resources/application.conf | 22 +- .../src/main/resources/application.conf.bak | 25 ++ .../src/main/webapp/META-INF/context.xml | 3 + .../src/main/webapp/WEB-INF/web.xml | 6 +- eagle-webservice/src/main/webapp/app/index.html | 2 +- .../webapp/app/partials/config/application.html | 8 +- .../webapp/app/partials/config/feature.html | 2 +- .../main/webapp/app/partials/config/site.html | 4 +- .../classification/page/sensitivity/folder.html | 2 +- .../app/public/feature/common/controller.js | 4 +- .../feature/common/page/policyDetail.html | 2 +- .../public/feature/common/page/policyEdit.html | 2 +- .../public/feature/common/page/policyList.html | 4 +- .../feature/metadata/page/streamList.html | 2 +- .../src/main/webapp/app/public/js/app.config.js | 2 +- .../webapp/app/public/js/srv/applicationSrv.js | 4 +- .../main/webapp/app/public/js/srv/siteSrv.js | 2 +- eagle-webservice/src/main/webapp/index.html | 2 +- pom.xml | 7 + 87 files changed, 1787 insertions(+), 478 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-assembly/src/main/bin/eagle-topology-init.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh index c326e38..fc476b4 100755 --- a/eagle-assembly/src/main/bin/eagle-topology-init.sh +++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh @@ -41,34 +41,34 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a echo "" echo "Importing application definitions ..." -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hdfsAuditLog"},"desc":"HDFS audit log security check application","alias":"HDFS","group":"DAM","features":["common","classification","userProfile","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"hdfsResource\"\n\t}\n}"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hdfsAuditLog"},"description":"HDFS audit log security check application","alias":"HDFS","groupName":"DAM","features":["common","classification","userProfile","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"hdfsResource\"\n\t}\n}"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hbaseSecurityLog"},"desc":"HBASE audit log security check application","alias":"HBASE","group":"DAM","features":["common","classification","userProfile","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"hbaseResourceSensitivity\",\n\t\t\"service\": \"HbaseResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"hbaseResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"table\",\n\t\t\"api\": {\n\t\t\t\"database\": \"hbaseResource/namespaces\",\n\t\t\t\"table\": \"hbaseResource/tables\",\n\t\t\t\"column\": \"hbaseResource/columns\"\n\t\t},\n\t\t\"mapping\": {\n\t\t\t\"database\": \"namespace\",\n\t\t\t\"table\": \"table\",\n\t\t\t\"column\": \"columnFamily\"\n\t\t}\n\t}\n}"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hbaseSecurityLog"},"description":"HBASE audit log security check application","alias":"HBASE","groupName":"DAM","features":["common","classification","userProfile","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"hbaseResourceSensitivity\",\n\t\t\"service\": \"HbaseResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"hbaseResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"table\",\n\t\t\"api\": {\n\t\t\t\"database\": \"hbaseResource/namespaces\",\n\t\t\t\"table\": \"hbaseResource/tables\",\n\t\t\t\"column\": \"hbaseResource/columns\"\n\t\t},\n\t\t\"mapping\": {\n\t\t\t\"database\": \"namespace\",\n\t\t\t\"table\": \"table\",\n\t\t\t\"column\": \"columnFamily\"\n\t\t}\n\t}\n}"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hiveQueryLog"},"desc":"Hive query log security check application","alias":"HIVE","group":"DAM","features":["common","classification","userProfile","metadata"], "config":"{\n\t\"view\": {\n\t\t\"prefix\": \"hiveResourceSensitivity\",\n\t\t\"service\": \"HiveResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"hiveResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"table\",\n\t\t\"api\": {\n\t\t\t\"database\": \"hiveResource/databases\",\n\t\t\t\"table\": \"hiveResource/tables\",\n\t\t\t\"column\": \"hiveResource/columns\"\n\t\t},\n\t\t\"mapping\": {\n\t\t\t\"database\": \"database\",\n\t\t\t\"table\": \"table\",\n\t\t\t\"column\": \"column\"\n\t\t}\n\t}\n}"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService"; -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hiveQueryLog"},"description":"Hive query log security check application","alias":"HIVE","groupName":"DAM","features":["common","classification","userProfile","metadata"], "config":"{\n\t\"view\": {\n\t\t\"prefix\": \"hiveResourceSensitivity\",\n\t\t\"service\": \"HiveResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"hiveResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"table\",\n\t\t\"api\": {\n\t\t\t\"database\": \"hiveResource/databases\",\n\t\t\t\"table\": \"hiveResource/tables\",\n\t\t\t\"column\": \"hiveResource/columns\"\n\t\t},\n\t\t\"mapping\": {\n\t\t\t\"database\": \"database\",\n\t\t\t\"table\": \"table\",\n\t\t\t\"column\": \"column\"\n\t\t}\n\t}\n}"}]' echo "" echo "Importing feature definitions ..." -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"common"},"desc":"Provide the Policy & Alert feature.","version":"v0.3.0"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"common"},"description":"Provide the Policy & Alert feature.","version":"v0.3.0"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"classification"},"desc":"Sensitivity browser of the data classification.","version":"v0.3.0"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"classification"},"description":"Sensitivity browser of the data classification.","version":"v0.3.0"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"userProfile"},"desc":"Machine learning of the user profile","version":"v0.3.0"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"userProfile"},"description":"Machine learning of the user profile","version":"v0.3.0"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metadata"},"desc":"Stream metadata viewer","version":"v0.3.0"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metadata"},"description":"Stream metadata viewer","version":"v0.3.0"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metrics"},"desc":"Metrics dashboard","version":"v0.3.0"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"; -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metrics"},"description":"Metrics dashboard","version":"v0.3.0"}]' ## AlertStreamService: alert streams generated from data source echo "" echo "Importing AlertStreamService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hdfsAuditLog","streamName":"hdfsAuditLogEventStream"},"desc":"alert event stream from hdfs audit log"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hdfsAuditLog","streamName":"hdfsAuditLogEventStream"},"description":"alert event stream from hdfs audit log"}]' ## AlertExecutorService: what alert streams are consumed by alert executor echo "" echo "Importing AlertExecutorService for HDFS... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hdfsAuditLog","alertExecutorId":"hdfsAuditLogAlertExecutor","streamName":"hdfsAuditLogEventStream"},"desc":"alert executor for hdfs audit log event stream"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hdfsAuditLog","alertExecutorId":"hdfsAuditLogAlertExecutor","streamName":"hdfsAuditLogEventStream"},"description":"alert executor for hdfs audit log event stream"}]' ## AlertStreamSchemaService: schema for event from alert stream echo "" @@ -83,13 +83,13 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a #### AlertStreamService: alert streams generated from data source echo "" echo "Importing AlertStreamService for HBASE... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hbaseSecurityLog","streamName":"hbaseSecurityLogEventStream"},"desc":"alert event stream from hbase security audit log"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hbaseSecurityLog","streamName":"hbaseSecurityLogEventStream"},"description":"alert event stream from hbase security audit log"}]' #### AlertExecutorService: what alert streams are consumed by alert executor echo "" echo "Importing AlertExecutorService for HBASE... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hbaseSecurityLog","alertExecutorId":"hbaseSecurityLogAlertExecutor","streamName":"hbaseSecurityLogEventStream"},"desc":"alert executor for hbase security log event stream"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hbaseSecurityLog","alertExecutorId":"hbaseSecurityLogAlertExecutor","streamName":"hbaseSecurityLogEventStream"},"description":"alert executor for hbase security log event stream"}]' #### AlertStreamSchemaService: schema for event from alert stream @@ -105,12 +105,12 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a ## AlertStreamService: alert streams generated from data source echo "" echo "Importing AlertStreamService for HIVE... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hiveQueryLog","streamName":"hiveAccessLogStream"},"desc":"alert event stream from hive query"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"; -d '[{"prefix":"alertStream","tags":{"application":"hiveQueryLog","streamName":"hiveAccessLogStream"},"description":"alert event stream from hive query"}]' ## AlertExecutorService: what alert streams are consumed by alert executor echo "" echo "Importing AlertExecutorService for HIVE... " -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hiveQueryLog","alertExecutorId":"hiveAccessAlertByRunningJob","streamName":"hiveAccessLogStream"},"desc":"alert executor for hive query log event stream"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"; -d '[{"prefix":"alertExecutor","tags":{"application":"hiveQueryLog","alertExecutorId":"hiveAccessAlertByRunningJob","streamName":"hiveAccessLogStream"},"description":"alert executor for hive query log event stream"}]' ## AlertStreamSchemaServiceService: schema for event from alert stream echo "" @@ -125,12 +125,12 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a echo "" echo "Importing AlertDefinitionService for USERPROFILE" curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H "Content-Type: application/json" "http://$EAGLE_SERVICE_HOST:$EAGLE_SERVICE_PORT/eagle-service/rest/entities?serviceName=AlertDefinitionService"; \ - -d '[ { "prefix": "alertdef", "tags": { "site": "sandbox", "application": "userProfile", "alertExecutorId": "userProfileAnomalyDetectionExecutor", "policyId": "userProfile", "policyType": "MachineLearning" }, "desc": "user profile anomaly detection", "policyDef": "{\"type\":\"MachineLearning\",\"alertContext\":{\"site\":\"sandbox\",\"application\":\"userProfile\",\"component\":\"testComponent\",\"description\":\"ML based user profile anomaly detection\",\"severity\":\"WARNING\",\"notificationByEmail\":\"true\"},\"algorithms\":[{\"name\":\"EigenDecomposition\",\"evaluator\":\"org.apache.eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator\",\"description\":\"EigenBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"},{\"name\":\"KDE\",\"evaluator\":\"org.apache.eagle.security.userprofile.impl.UserProfileAnomalyKDEEvaluator\",\"description\":\"Dens ityBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"}]}", "dedupeDef": "{\"alertDedupIntervalMin\":\"0\",\"emailDedupIntervalMin\":\"0\"}", "notificationDef": "", "remediationDef": "", "enabled": true } ]' + -d '[ { "prefix": "alertdef", "tags": { "site": "sandbox", "application": "userProfile", "alertExecutorId": "userProfileAnomalyDetectionExecutor", "policyId": "userProfile", "policyType": "MachineLearning" }, "description": "user profile anomaly detection", "policyDef": "{\"type\":\"MachineLearning\",\"alertContext\":{\"site\":\"sandbox\",\"application\":\"userProfile\",\"component\":\"testComponent\",\"description\":\"ML based user profile anomaly detection\",\"severity\":\"WARNING\",\"notificationByEmail\":\"true\"},\"algorithms\":[{\"name\":\"EigenDecomposition\",\"evaluator\":\"org.apache.eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator\",\"description\":\"EigenBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"},{\"name\":\"KDE\",\"evaluator\":\"org.apache.eagle.security.userprofile.impl.UserProfileAnomalyKDEEvaluator\",\"description\" :\"DensityBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"}]}", "dedupeDef": "{\"alertDedupIntervalMin\":\"0\",\"emailDedupIntervalMin\":\"0\"}", "notificationDef": "", "remediationDef": "", "enabled": true } ]' echo "" echo "Importing AlertExecutorService for USERPROFILE" curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H "Content-Type: application/json" "http://$EAGLE_SERVICE_HOST:$EAGLE_SERVICE_PORT/eagle-service/rest/entities?serviceName=AlertExecutorService"; \ - -d '[ { "prefix": "alertExecutor", "tags":{ "site":"sandbox", "application":"userProfile", "alertExecutorId" : "userProfileAnomalyDetectionExecutor", "streamName":"userActivity" }, "desc": "user activity data source" } ]' + -d '[ { "prefix": "alertExecutor", "tags":{ "site":"sandbox", "application":"userProfile", "alertExecutorId" : "userProfileAnomalyDetectionExecutor", "streamName":"userActivity" }, "description": "user activity data source" } ]' echo "" echo "Importing AlertStreamService for USERPROFILE" http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-assembly/src/main/conf/eagle-service.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/eagle-service.conf b/eagle-assembly/src/main/conf/eagle-service.conf index 8e2b96a..9b97545 100644 --- a/eagle-assembly/src/main/conf/eagle-service.conf +++ b/eagle-assembly/src/main/conf/eagle-service.conf @@ -13,30 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# eagle configuration -eagle{ - # eagle web service configuration - service{ - # storage type: ["hbase","jdbc"] - # default is "hbase" - storage-type="hbase" - - # hbase configuration: hbase.zookeeper.quorum - # default is "localhost" - hbase-zookeeper-quorum="localhost" - - # hbase configuration: hbase.zookeeper.property.clientPort - # default is 2181 - hbase-zookeeper-property-clientPort=2181 - - # hbase configuration: zookeeper.znode.parent - # default is "/hbase" - zookeeper-znode-parent="/hbase-unsecure" - - # eagle web login profile: [sandbox, default] - # default is sandbox - springActiveProfile="sandbox" - - audit-enabled=false +eagle { + service { + storage-type="jdbc" + storage-adapter="derby" + storage-username="eagle" + storage-password=eagle + storage-database=eagle + # Derby database location: $TOMCAT_HOME/data/eagle + storage-connection-url="jdbc:derby:data/eagle;create=true" + storage-connection-props="encoding=UTF-8" + storage-driver-class="org.apache.derby.jdbc.EmbeddedDriver" + storage-connection-max=8 } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-assembly/src/main/lib/jdbc/eagle-jdbc-mysql.sql ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/lib/jdbc/eagle-jdbc-mysql.sql b/eagle-assembly/src/main/lib/jdbc/eagle-jdbc-mysql.sql new file mode 100644 index 0000000..d4be66c --- /dev/null +++ b/eagle-assembly/src/main/lib/jdbc/eagle-jdbc-mysql.sql @@ -0,0 +1,331 @@ +-- /* +-- * Licensed to the Apache Software Foundation (ASF) under one or more +-- * contributor license agreements. See the NOTICE file distributed with +-- * this work for additional information regarding copyright ownership. +-- * The ASF licenses this file to You under the Apache License, Version 2.0 +-- * (the "License"); you may not use this file except in compliance with +-- * the License. You may obtain a copy of the License at +-- * +-- * http://www.apache.org/licenses/LICENSE-2.0 +-- * +-- * Unless required by applicable law or agreed to in writing, software +-- * distributed under the License is distributed on an "AS IS" BASIS, +-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- * See the License for the specific language governing permissions and +-- * limitations under the License. +-- * +-- */ + +-- MySQL dump 10.13 Distrib 5.6.23, for osx10.8 (x86_64) +-- +-- Host: localhost Database: eagle +-- ------------------------------------------------------ +-- Server version 5.6.23 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `alertdef_alertdef` +-- + +DROP TABLE IF EXISTS `alertdef_alertdef`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `alertdef_alertdef` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `site` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `alertexecutorid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `policyid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `policytype` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(200) COLLATE utf8_bin DEFAULT NULL, + `policydef` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `dedupedef` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `notificationdef` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `remediationdef` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `owner` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `lastmodifieddate` bigint(20) DEFAULT NULL, + `severity` bigint(20) DEFAULT NULL, + `createdtime` bigint(20) DEFAULT NULL, + `markdownReason` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `markdownEnabled` tinyint(1) DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `alertdef_alertdef` +-- + +LOCK TABLES `alertdef_alertdef` WRITE; +/*!40000 ALTER TABLE `alertdef_alertdef` DISABLE KEYS */; +/*!40000 ALTER TABLE `alertdef_alertdef` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `alertdetail_hadoop` +-- + +DROP TABLE IF EXISTS `alertdetail_hadoop`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `alertdetail_hadoop` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `site` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `hostname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `policyid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `alertsource` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `sourcestreams` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `alertexecutorid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `remediationid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `remediationcallback` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `alertcontext` varchar(1000) COLLATE utf8_bin DEFAULT NULL, + `streamid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `alertdetail_hadoop` +-- + +LOCK TABLES `alertdetail_hadoop` WRITE; +/*!40000 ALTER TABLE `alertdetail_hadoop` DISABLE KEYS */; +/*!40000 ALTER TABLE `alertdetail_hadoop` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `alertexecutor_alertexecutor` +-- + +DROP TABLE IF EXISTS `alertexecutor_alertexecutor`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `alertexecutor_alertexecutor` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `alertexecutorid` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `streamname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `site` varchar(45) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `alertexecutor_alertexecutor` +-- + +LOCK TABLES `alertexecutor_alertexecutor` WRITE; +/*!40000 ALTER TABLE `alertexecutor_alertexecutor` DISABLE KEYS */; +INSERT INTO `alertexecutor_alertexecutor` VALUES ('0ijKT3_____62aP_uMZ-K1SsoVDrH3vKa382HVykBVAJItDs',0,'hiveQueryLog','hiveAccessAlertByRunningJob','hiveAccessLogStream','alert executor for hive query log event stream',NULL),('0ijKT3_____62aP_uMZ-K2-GR_rrH3vKDuSvMwA130dvL77HXKQFUNuQa14',0,'userProfile','userProfileAnomalyDetectionExecutor','userActivity','user activity data source','sandbox'),('0ijKT3_____62aP_uMZ-K4uAAAjrH3vKhK_cHVykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogAlertExecutor','hdfsAuditLogEventStream','alert executor for hdfs audit log event stream',NULL),('0ijKT3_____62aP_uMZ-K_85ls_rH3vK8F7dJFykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogAlertExecutor','hbaseSecurityLogEventStream','alert executor for hbase security log event stream',NULL); +/*!40000 ALTER TABLE `alertexecutor_alertexecutor` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `alertstream_alertstream` +-- + +DROP TABLE IF EXISTS `alertstream_alertstream`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `alertstream_alertstream` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `streamname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + `site` varchar(45) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `alertstream_alertstream` +-- + +LOCK TABLES `alertstream_alertstream` WRITE; +/*!40000 ALTER TABLE `alertstream_alertstream` DISABLE KEYS */; +INSERT INTO `alertstream_alertstream` VALUES ('x3ZP_H_____62aP_uMZ-K2-GR_oANd9Hby--x1ykBVDbkGte',0,'userProfile','userActivity',NULL,'sandbox'); +/*!40000 ALTER TABLE `alertstream_alertstream` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `alertstreamschema_alertstreamschema` +-- + +DROP TABLE IF EXISTS `alertstreamschema_alertstreamschema`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `alertstreamschema_alertstreamschema` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `streamname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `attrname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `attrtype` varchar(20) COLLATE utf8_bin DEFAULT NULL, + `category` varchar(20) COLLATE utf8_bin DEFAULT NULL, + `attrValueResolver` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `usedastag` tinyint(1) DEFAULT NULL, + `attrdescription` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + `attrdisplayname` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `defaultvalue` varchar(100) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `alertstreamschema_alertstreamschema` +-- + +LOCK TABLES `alertstreamschema_alertstreamschema` WRITE; +/*!40000 ALTER TABLE `alertstreamschema_alertstreamschema` DISABLE KEYS */; +INSERT INTO `alertstreamschema_alertstreamschema` VALUES ('iSeEvX_____62aP_uMZ-K1SsoVAhAmgc66vEDlykBVAJItDs',0,'hiveQueryLog','hiveAccessLogStream','resource','string','','org.apache.eagle.service.security.hive.resolver.HiveMetadataResolver',NULL,'/database/table/column or /database/table/*',NULL,NULL),('iSeEvX_____62aP_uMZ-K1SsoVAhAmgcA0kpFlykBVAJItDs',0,'hiveQueryLog','hiveAccessLogStream','timestamp','long','','',NULL,'milliseconds of the datetime',NULL,NULL),('iSeEvX_____62aP_uMZ-K1SsoVAhAmgcADbry1ykBVAJItDs',0,'hiveQueryLog','hiveAccessLogStream','user','string','','',NULL,'process user',NULL,NULL),('iSeEvX_____62aP_uMZ-K1SsoVAhAmgcOKXfS1ykBVAJItDs',0,'hiveQueryLog','hiveAccessLogStream','command','string','','org.apache.eagle.service.security.hive.resolver.HiveCommandResolver',NULL,'hive sql command, such as SELECT, INSERT and DELETE',NULL,NULL),('iSeEvX_____62aP_uMZ-K1SsoVAhAmgcX026eVykBVAJItDs',0,'hiveQueryLog','hiveAccessLogStream','sensitivityType','string','','org.apache. eagle.service.security.hive.resolver.HiveSensitivityTypeResolver',NULL,'mark such as PHONE_NUMBER',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcA0kpFlykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','timestamp','long','','',NULL,'milliseconds of the datetime',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcAAG95FykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','src','string','','org.apache.eagle.service.security.hdfs.resolver.HDFSResourceResolver',NULL,'source directory or file, such as /tmp',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcAAGBOlykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','cmd','string','','org.apache.eagle.service.security.hdfs.resolver.HDFSCommandResolver',NULL,'file/directory operation, such as getfileinfo, open, listStatus and so on',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcAAGFxVykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','dst','string','','org.apache.eagle.service.security.hdfs.resolver.HDFSResourceResolver',NULL,'destination di rectory, such as /tmp',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcADD1qFykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','host','string','','',NULL,'hostname, such as localhost',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcADbry1ykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','user','string','','',NULL,'process user',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcMC9vDFykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','securityZone','string','','',NULL,'',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcX026eVykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','sensitivityType','string','','org.apache.eagle.service.security.hdfs.resolver.HDFSSensitivityTypeResolver',NULL,'mark such as AUDITLOG, SECURITYLOG',NULL,NULL),('iSeEvX_____62aP_uMZ-K4uAAAghAmgcya4BqFykBVB3iZNS',0,'hdfsAuditLog','hdfsAuditLogEventStream','allowed','bool','','',NULL,'true, false or none',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcA0kpFlykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventS tream','timestamp','long','','',NULL,'milliseconds of the datetime',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcADD1qFykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','host','string','','',NULL,'remote ip address to access hbase',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcADbry1ykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','user','string','','',NULL,'hbase user',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcBoM-VFykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','scope','string','','org.apache.eagle.service.security.hbase.resolver.HbaseMetadataResolver',NULL,'the resources which users are then granted specific permissions (Read, Write, Execute, Create, Admin) against',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcQU7yj1ykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','request','string','','org.apache.eagle.service.security.hbase.resolver.HbaseRequestResolver',NULL,'',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcX026eVykBVCMC nLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','sensitivityType','string','','org.apache.eagle.service.security.hbase.resolver.HbaseSensitivityTypeResolver',NULL,'',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcqy9-NlykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','action','string','','org.apache.eagle.service.security.hbase.resolver.HbaseActionResolver',NULL,'action types, such as read, write, create, execute, and admin',NULL,NULL),('iSeEvX_____62aP_uMZ-K_85ls8hAmgcys3P8lykBVCMCnLr',0,'hbaseSecurityLog','hbaseSecurityLogEventStream','status','string','','',NULL,'access status: allowed or denied',NULL,NULL); +/*!40000 ALTER TABLE `alertstreamschema_alertstreamschema` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `eagleapplicationdesc_eagleapplicationdesc` +-- + +DROP TABLE IF EXISTS `eagleapplicationdesc_eagleapplicationdesc`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `eagleapplicationdesc_eagleapplicationdesc` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + `alias` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `groupName` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `features` blob, + `config` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `eagleapplicationdesc_eagleapplicationdesc` +-- + +LOCK TABLES `eagleapplicationdesc_eagleapplicationdesc` WRITE; +/*!40000 ALTER TABLE `eagleapplicationdesc_eagleapplicationdesc` DISABLE KEYS */; +INSERT INTO `eagleapplicationdesc_eagleapplicationdesc` VALUES ('54TRXX_____62aP_XKQFUAki0Ow',0,'hiveQueryLog','Hive query log security check application','HIVE','DAM','\0\0\0\0\0\0\0\0\0common\0\0\0\0\0\0classification\0\0\0\0\0\0 userProfile\0\0\0\0\0\0metadata','{\n \"view\": {\n \"prefix\": \"hiveResourceSensitivity\",\n \"service\": \"HiveResourceSensitivityService\",\n \"keys\": [\n \"hiveResource\",\n \"sensitivityType\"\n ],\n \"type\": \"table\",\n \"api\": {\n \"database\": \"hiveResource/databases\",\n \"table\": \"hiveResource/tables\",\n \"column\": \"hiveResource/columns\"\n },\n \"mapping\": {\n \"database\": \"database\",\n \"table\": \"table\",\n \"column\": \"column\"\n }\n }\n}'),('54TRXX_____62aP_XKQFUHeJk1I',0,'hdfsAuditLog','HDFS audit log security check application','HDFS','DAM','\0\0\0\0\0\0\0\0\0common\0\0\0\0\0\0classification\0\0\0\0\0\0 userProfile\0\0\0\0\0\0metadata','{\n \"view\": {\n \"prefix\": \"fileSensitiv ity\",\n \"service\": \"FileSensitivityService\",\n \"keys\": [\n \"filedir\",\n \"sensitivityType\"\n ],\n \"type\": \"folder\",\n \"api\": \"hdfsResource\"\n }\n}'),('54TRXX_____62aP_XKQFUIwKcus',0,'hbaseSecurityLog','HBASE audit log security check application','HBASE','DAM','\0\0\0\0\0\0\0\0\0common\0\0\0\0\0\0classification\0\0\0\0\0\0 userProfile\0\0\0\0\0\0metadata','{\n \"view\": {\n \"prefix\": \"hbaseResourceSensitivity\",\n \"service\": \"HbaseResourceSensitivityService\",\n \"keys\": [\n \"hbaseResource\",\n \"sensitivityType\"\n ],\n \"type\": \"table\",\n \"api\": {\n \"database\": \"hbaseResource/namespaces\",\n \"table\": \"hbaseResource/tables\",\n \"column\": \"hbaseResource/columns\"\n },\n \"mapping\": {\n \"database\": \"namespace\",\n \"table\": \"table\",\n \"column\": \"columnFamily\"\n }\n }\n}'); +/*!40000 ALTER TABLE `eagleapplicationdesc_eagleapplicationdesc` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `eaglefeaturedesc_eaglefeaturedesc` +-- + +DROP TABLE IF EXISTS `eaglefeaturedesc_eaglefeaturedesc`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `eaglefeaturedesc_eaglefeaturedesc` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `feature` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `description` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + `version` varchar(100) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `eaglefeaturedesc_eaglefeaturedesc` +-- + +LOCK TABLES `eaglefeaturedesc_eaglefeaturedesc` WRITE; +/*!40000 ALTER TABLE `eaglefeaturedesc_eaglefeaturedesc` DISABLE KEYS */; +INSERT INTO `eaglefeaturedesc_eaglefeaturedesc` VALUES ('4DMSA3_____62aP_xaJ69hbKM-Y',0,'classification','Sensitivity browser of the data classification.','v0.3.0'),('4DMSA3_____62aP_xaJ69jj4wMM',0,'metrics','Metrics dashboard','v0.3.0'),('4DMSA3_____62aP_xaJ69q8_Kes',0,'common','Provide the Policy & Alert feature.','v0.3.0'),('4DMSA3_____62aP_xaJ69tuQa14',0,'userProfile','Machine learning of the user profile','v0.3.0'),('4DMSA3_____62aP_xaJ69uUtey8',0,'metadata','Stream metadata viewer','v0.3.0'); +/*!40000 ALTER TABLE `eaglefeaturedesc_eaglefeaturedesc` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `eaglesiteapplication_eaglesiteapplication` +-- + +DROP TABLE IF EXISTS `eaglesiteapplication_eaglesiteapplication`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `eaglesiteapplication_eaglesiteapplication` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `site` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `application` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + `config` varchar(1024) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `eaglesiteapplication_eaglesiteapplication` +-- + +LOCK TABLES `eaglesiteapplication_eaglesiteapplication` WRITE; +/*!40000 ALTER TABLE `eaglesiteapplication_eaglesiteapplication` DISABLE KEYS */; +INSERT INTO `eaglesiteapplication_eaglesiteapplication` VALUES ('D-7M5X_____62aP_ADXfR28vvsdcpAVQCSLQ7A',0,'sandbox','hiveQueryLog',1,'{\"accessType\":\"metastoredb_jdbc\",\"password\":\"hive\",\"user\":\"hive\",\"jdbcDriverClassName\":\"com.mysql.jdbc.Driver\",\"jdbcUrl\":\"jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true\"}'),('D-7M5X_____62aP_ADXfR28vvsdcpAVQd4mTUg',0,'sandbox','hdfsAuditLog',1,'{\"fs.defaultFS\":\"hdfs://sandbox.hortonworks.com:8020\"}'),('D-7M5X_____62aP_ADXfR28vvsdcpAVQjApy6w',0,'sandbox','hbaseSecurityLog',1,'{\"hbase.zookeeper.property.clientPort\":\"2181\", \"hbase.zookeeper.quorum\":\"localhost\"}'); +/*!40000 ALTER TABLE `eaglesiteapplication_eaglesiteapplication` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `eaglesitedesc_eaglesitedesc` +-- + +DROP TABLE IF EXISTS `eaglesitedesc_eaglesitedesc`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `eaglesitedesc_eaglesitedesc` ( + `uuid` varchar(100) COLLATE utf8_bin NOT NULL, + `timestamp` bigint(20) DEFAULT NULL, + `site` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `enabled` tinyint(1) DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `uuid_UNIQUE` (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `eaglesitedesc_eaglesitedesc` +-- + +LOCK TABLES `eaglesitedesc_eaglesitedesc` WRITE; +/*!40000 ALTER TABLE `eaglesitedesc_eaglesitedesc` DISABLE KEYS */; +INSERT INTO `eaglesitedesc_eaglesitedesc` VALUES ('phJknH_____62aP_ADXfR28vvsc',0,'sandbox',1); +/*!40000 ALTER TABLE `eaglesitedesc_eaglesitedesc` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2016-03-08 18:34:19 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/main/java/org/apache/eagle/notification/email/AlertEmailGenerator.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/main/java/org/apache/eagle/notification/email/AlertEmailGenerator.java b/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/main/java/org/apache/eagle/notification/email/AlertEmailGenerator.java index ddd3c96..fd6b794 100644 --- a/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/main/java/org/apache/eagle/notification/email/AlertEmailGenerator.java +++ b/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/main/java/org/apache/eagle/notification/email/AlertEmailGenerator.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; +import org.apache.eagle.common.metric.AlertContext; import org.apache.eagle.policy.common.Constants; import org.apache.eagle.alert.entity.AlertAPIEntity; import com.typesafe.config.ConfigObject; @@ -55,12 +56,13 @@ public class AlertEmailGenerator{ AlertEmailContext email = new AlertEmailContext(); AlertEmailComponent component = new AlertEmailComponent(); - component.setAlertContext(entity.getAlertContext()); + AlertContext context = AlertContext.fromJsonString(entity.getAlertContext()); + component.setAlertContext(context); List<AlertEmailComponent> components = new ArrayList<AlertEmailComponent>(); components.add(component); email.setComponents(components); - if (entity.getAlertContext().getProperty(Constants.SUBJECT) != null) { - email.setSubject(entity.getAlertContext().getProperty(Constants.SUBJECT)); + if (context.getProperty(Constants.SUBJECT) != null) { + email.setSubject(context.getProperty(Constants.SUBJECT)); } else email.setSubject(subject); email.setVelocityTplFile(tplFile); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/test/java/org/apache/eagle/notifications/testcases/TestAlertKafkaPlugin.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/test/java/org/apache/eagle/notifications/testcases/TestAlertKafkaPlugin.java b/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/test/java/org/apache/eagle/notifications/testcases/TestAlertKafkaPlugin.java index b5ed63e..318df22 100644 --- a/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/test/java/org/apache/eagle/notifications/testcases/TestAlertKafkaPlugin.java +++ b/eagle-core/eagle-alert/eagle-alert-notification-plugin/src/test/java/org/apache/eagle/notifications/testcases/TestAlertKafkaPlugin.java @@ -24,11 +24,7 @@ import org.apache.eagle.alert.entity.AlertAPIEntity; import org.apache.eagle.alert.entity.AlertDefinitionAPIEntity; import org.apache.eagle.common.metric.AlertContext; import org.apache.eagle.notification.plugin.AlertKafkaPlugin; -import org.apache.eagle.notification.plugin.KafkaProducerSingleton; -import org.apache.eagle.notification.utils.NotificationPluginUtils; import org.apache.eagle.policy.common.Constants; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.Ignore; import org.junit.Test; @@ -53,7 +49,7 @@ public class TestAlertKafkaPlugin { alert.setTags(new HashMap<String, String>()); alert.getTags().put(Constants.POLICY_ID, "testPolicyId"); alert.setDescription(""); - alert.setAlertContext(new AlertContext()); + alert.setAlertContext(new AlertContext().toJsonString()); plugin.onAlert(alert); Thread.sleep(1000); // wait for message sent out Assert.assertTrue(plugin.getStatus().successful); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/notification/AlertEmailGenerator.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/notification/AlertEmailGenerator.java b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/notification/AlertEmailGenerator.java new file mode 100644 index 0000000..ef45af3 --- /dev/null +++ b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/notification/AlertEmailGenerator.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * + */ +package org.apache.eagle.alert.notification; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +import org.apache.eagle.common.metric.AlertContext; +import org.apache.eagle.policy.common.Constants; +import org.apache.eagle.alert.common.AlertEmailSender; +import org.apache.eagle.alert.email.AlertEmailComponent; +import org.apache.eagle.alert.email.AlertEmailContext; +import org.apache.eagle.alert.entity.AlertAPIEntity; +import com.typesafe.config.ConfigObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AlertEmailGenerator{ + private String tplFile; + private String sender; + private String recipients; + private String subject; + private ConfigObject eagleProps; + + private ThreadPoolExecutor executorPool; + + private final static Logger LOG = LoggerFactory.getLogger(AlertEmailGenerator.class); + + private final static long MAX_TIMEOUT_MS =60000; + + public void sendAlertEmail(AlertAPIEntity entity) { + sendAlertEmail(entity, recipients, null); + } + + public void sendAlertEmail(AlertAPIEntity entity, String recipients) { + sendAlertEmail(entity, recipients, null); + } + + public void sendAlertEmail(AlertAPIEntity entity, String recipients, String cc) { + AlertEmailContext email = new AlertEmailContext(); + + AlertEmailComponent component = new AlertEmailComponent(); + component.setAlertContext(AlertContext.fromJsonString(entity.getAlertContext())); + List<AlertEmailComponent> components = new ArrayList<AlertEmailComponent>(); + components.add(component); + email.setComponents(components); + if (AlertContext.fromJsonString(entity.getAlertContext()).getProperty(Constants.SUBJECT) != null) { + email.setSubject(AlertContext.fromJsonString(entity.getAlertContext()).getProperty(Constants.SUBJECT)); + } + else email.setSubject(subject); + email.setVelocityTplFile(tplFile); + email.setRecipients(recipients); + email.setCc(cc); + email.setSender(sender); + + /** asynchronized email sending */ + @SuppressWarnings("rawtypes") + AlertEmailSender thread = new AlertEmailSender(email, eagleProps); + + if(this.executorPool == null) throw new IllegalStateException("Invoking thread executor pool but it's is not set yet"); + + LOG.info("Sending email in asynchronous to: "+recipients+", cc: "+cc); + Future future = this.executorPool.submit(thread); + try { + future.get(MAX_TIMEOUT_MS, TimeUnit.MILLISECONDS); + LOG.info(String.format("Successfully send email to %s", recipients)); + } catch (InterruptedException | ExecutionException e) { + LOG.error(String.format("Failed to send email to %s, due to:%s",recipients,e),e); + } catch (TimeoutException e) { + LOG.error(String.format("Failed to send email to %s due to timeout exception, max timeout: %s ms ",recipients, MAX_TIMEOUT_MS),e); + } + } + + public String getTplFile() { + return tplFile; + } + + public void setTplFile(String tplFile) { + this.tplFile = tplFile; + } + + public String getSender() { + return sender; + } + + public void setSender(String sender) { + this.sender = sender; + } + + public String getRecipients() { + return recipients; + } + + public void setRecipients(String recipients) { + this.recipients = recipients; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public ConfigObject getEagleProps() { + return eagleProps; + } + + public void setEagleProps(ConfigObject eagleProps) { + this.eagleProps = eagleProps; + } + + public void setExecutorPool(ThreadPoolExecutor executorPool) { + this.executorPool = executorPool; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/siddhi/SiddhiAlertAPIEntityRender.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/siddhi/SiddhiAlertAPIEntityRender.java b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/siddhi/SiddhiAlertAPIEntityRender.java index d1e1bdc..c7ff74c 100644 --- a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/siddhi/SiddhiAlertAPIEntityRender.java +++ b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/siddhi/SiddhiAlertAPIEntityRender.java @@ -103,7 +103,7 @@ public class SiddhiAlertAPIEntityRender implements ResultRender<AlertDefinitionA context.addProperty(Constants.POLICY_DETAIL_URL, UrlBuilder.buiildPolicyDetailUrl(host, port, tags)); context.addProperty(Constants.ALERT_DETAIL_URL, UrlBuilder.buildAlertDetailUrl(host, port, entity)); - entity.setAlertContext(context); + entity.setAlertContext(context.toJsonString()); return entity; } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-alert/eagle-alert-process/src/test/java/org/apache/eagle/alert/state/TestSiddhiStateSnapshotAndRestore.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert/eagle-alert-process/src/test/java/org/apache/eagle/alert/state/TestSiddhiStateSnapshotAndRestore.java b/eagle-core/eagle-alert/eagle-alert-process/src/test/java/org/apache/eagle/alert/state/TestSiddhiStateSnapshotAndRestore.java index bd65b9b..6d8065f 100644 --- a/eagle-core/eagle-alert/eagle-alert-process/src/test/java/org/apache/eagle/alert/state/TestSiddhiStateSnapshotAndRestore.java +++ b/eagle-core/eagle-alert/eagle-alert-process/src/test/java/org/apache/eagle/alert/state/TestSiddhiStateSnapshotAndRestore.java @@ -275,6 +275,7 @@ public class TestSiddhiStateSnapshotAndRestore { inputHandler.send(new Object[]{curTime + 1000, "user", "open"}); inputHandler.send(new Object[]{curTime + 2000, "user", "open"}); inputHandler.send(new Object[]{curTime + 3000, "user", "open"}); + Thread.sleep(1000); byte[] state = executionPlanRuntime.snapshot(); int length = state.length; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java index a889361..1e8c013 100644 --- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java +++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java @@ -101,7 +101,7 @@ public class MLAnomalyCallbackImpl implements MLAnomalyCallback { context.addProperty(EagleConfigConstants.SITE, site); context.addProperty(Constants.POLICY_NAME, alertContext.policyId); - entity.setAlertContext(context); + entity.setAlertContext(context.toJsonString()); return entity; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertAPIEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertAPIEntity.java index 0e5d3c8..e4a64ab 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertAPIEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertAPIEntity.java @@ -16,15 +16,10 @@ */ package org.apache.eagle.alert.entity; +import org.apache.eagle.log.entity.meta.*; import org.apache.eagle.policy.common.Constants; import org.apache.eagle.common.metric.AlertContext; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; -import org.apache.eagle.log.entity.meta.Column; -import org.apache.eagle.log.entity.meta.ColumnFamily; -import org.apache.eagle.log.entity.meta.Prefix; -import org.apache.eagle.log.entity.meta.Service; -import org.apache.eagle.log.entity.meta.Table; -import org.apache.eagle.log.entity.meta.TimeSeries; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -35,6 +30,7 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @Service(Constants.ALERT_SERVICE_ENDPOINT_NAME) @TimeSeries(true) @JsonIgnoreProperties(ignoreUnknown = true) +@Tags({"site", "hostname", "application", "policyId", "sourceStreams", "alertSource", "alertExecutorId"}) public class AlertAPIEntity extends TaggedLogAPIEntity{ @Column("description") private String description; @@ -43,7 +39,7 @@ public class AlertAPIEntity extends TaggedLogAPIEntity{ @Column("remediationCallback") private String remediationCallback; @Column("alertContext") - private AlertContext alertContext; + private String alertContext; @Column("streamId") private String streamId; @@ -74,15 +70,24 @@ public class AlertAPIEntity extends TaggedLogAPIEntity{ _pcs.firePropertyChange("remediationCallback", null, null); } - public AlertContext getAlertContext() { + public String getAlertContext() { return alertContext; } + + public AlertContext getWrappedAlertContext() { + return AlertContext.fromJsonString(alertContext); + } - public void setAlertContext(AlertContext alertContext) { + public void setAlertContext(String alertContext) { this.alertContext = alertContext; _pcs.firePropertyChange("alertContext", null, null); } + public void setAlertContext(AlertContext alertContext) { + if(alertContext != null) this.alertContext = alertContext.toJsonString(); + _pcs.firePropertyChange("alertContext", null, null); + } + public String getStreamId() { return streamId; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDataSourceEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDataSourceEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDataSourceEntity.java index 0778ffc..7f8dbe3 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDataSourceEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDataSourceEntity.java @@ -37,7 +37,7 @@ public class AlertDataSourceEntity extends TaggedLogAPIEntity{ @Column("b") private String config; @Column("c") - private String desc; + private String description; public String getConfig() { return config; @@ -57,12 +57,12 @@ public class AlertDataSourceEntity extends TaggedLogAPIEntity{ valueChanged("enabled"); } - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; + public void setDescription(String desc) { + this.description = desc; valueChanged("desc"); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDefinitionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDefinitionAPIEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDefinitionAPIEntity.java index 98dc704..036ec7a 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDefinitionAPIEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertDefinitionAPIEntity.java @@ -52,7 +52,7 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; }) public class AlertDefinitionAPIEntity extends AbstractPolicyDefinitionEntity { @Column("a") - private String desc; + private String description; @Column("b") private String policyDef; @Column("c") @@ -76,13 +76,14 @@ public class AlertDefinitionAPIEntity extends AbstractPolicyDefinitionEntity { @Column("l") private String markdownReason; - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; - valueChanged("desc"); + public void setDescription(String desc) { + this.description = desc; + valueChanged("description"); } + public String getPolicyDef() { return policyDef; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertExecutorEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertExecutorEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertExecutorEntity.java index 692450f..7e023e6 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertExecutorEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertExecutorEntity.java @@ -32,17 +32,17 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @Service(Constants.ALERT_EXECUTOR_SERVICE_ENDPOINT_NAME) @JsonIgnoreProperties(ignoreUnknown = true) @TimeSeries(false) -@Tags({"application", "alertExecutorId", "streamName"}) +@Tags({"site","application", "alertExecutorId", "streamName"}) public class AlertExecutorEntity extends TaggedLogAPIEntity{ @Column("a") - private String desc; + private String description; - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; - valueChanged("desc"); + public void setDescription(String desc) { + this.description = desc; + valueChanged("description"); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertStreamEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertStreamEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertStreamEntity.java index eedbbf7..685da4f 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertStreamEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/AlertStreamEntity.java @@ -36,16 +36,16 @@ import org.apache.eagle.log.entity.meta.TimeSeries; @Service(Constants.ALERT_STREAM_SERVICE_ENDPOINT_NAME) @JsonIgnoreProperties(ignoreUnknown = true) @TimeSeries(false) -@Tags({"application", "streamName"}) +@Tags({"site","application", "streamName"}) public class AlertStreamEntity extends TaggedLogAPIEntity{ @Column("a") - private String desc; + private String description; - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; - valueChanged("desc"); + public void setDescription(String description) { + this.description = description; + valueChanged("description"); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/ApplicationDescServiceEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/ApplicationDescServiceEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/ApplicationDescServiceEntity.java index 76a2500..54ff203 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/ApplicationDescServiceEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/ApplicationDescServiceEntity.java @@ -37,11 +37,11 @@ import java.util.List; @Tags({"application"}) public class ApplicationDescServiceEntity extends TaggedLogAPIEntity { @Column("a") - private String desc; + private String description; @Column("b") private String alias; @Column("c") - private String group; + private String groupName; @Column("d") private List<String> features; @Column("e") @@ -56,22 +56,22 @@ public class ApplicationDescServiceEntity extends TaggedLogAPIEntity { valueChanged("config"); } - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; - valueChanged("desc"); + public void setDescription(String description) { + this.description = description; + valueChanged("description"); } - public String getGroup() { - return group; + public String getGroupName() { + return groupName; } - public void setGroup(String group) { - this.group = group; - valueChanged("group"); + public void setGroupName(String groupName) { + this.groupName = groupName; + valueChanged("groupName"); } public String getAlias() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/FeatureDescServiceEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/FeatureDescServiceEntity.java b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/FeatureDescServiceEntity.java index 4fc6dac..0f3c738 100644 --- a/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/FeatureDescServiceEntity.java +++ b/eagle-core/eagle-policy/eagle-policy-base/src/main/java/org/apache/eagle/alert/entity/FeatureDescServiceEntity.java @@ -35,7 +35,7 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @Tags({"feature"}) public class FeatureDescServiceEntity extends TaggedLogAPIEntity { @Column("a") - private String desc; + private String description; @Column("b") private String version; @@ -48,12 +48,12 @@ public class FeatureDescServiceEntity extends TaggedLogAPIEntity { valueChanged("version"); } - public String getDesc() { - return desc; + public String getDescription() { + return description; } - public void setDesc(String desc) { - this.desc = desc; - valueChanged("desc"); + public void setDescription(String description) { + this.description = description; + valueChanged("description"); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java index 58794ca..dd3a4c6 100644 --- a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java +++ b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java @@ -58,7 +58,7 @@ public class DateTimeUtil { public static String millisecondsToHumanDateWithSeconds(long milliseconds){ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// sdf.setTimeZone(CURRENT_TIME_ZONE); + sdf.setTimeZone(CURRENT_TIME_ZONE); Date t = new Date(); t.setTime(milliseconds); return sdf.format(t); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java index ed04f75..3e26f96 100755 --- a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java +++ b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java @@ -61,7 +61,9 @@ public class EagleConfigFactory implements EagleConfig { private EagleConfigFactory(){ init(); - this.pool = new HTablePool(this.hbaseConf, 10); + if(this.getStorageType() == null || this.getStorageType().equalsIgnoreCase("hbase")) { + this.pool = new HTablePool(this.hbaseConf, 10); + } } public static EagleConfig load(){ http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/metric/AlertContext.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/metric/AlertContext.java b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/metric/AlertContext.java index 0d1faa1..e046edf 100644 --- a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/metric/AlertContext.java +++ b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/metric/AlertContext.java @@ -16,6 +16,10 @@ */ package org.apache.eagle.common.metric; +import org.apache.log4j.spi.LoggerFactory; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -24,6 +28,8 @@ import java.util.Map; * not thread safe */ public class AlertContext implements Serializable{ + private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(AlertContext.class); + private Map<String, String> properties = new HashMap<String, String>(); public AlertContext(){ @@ -47,7 +53,30 @@ public class AlertContext implements Serializable{ this.properties.putAll(propHash); return this; } - + + public String toJsonString(){ + ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.writeValueAsString(properties); + }catch(Exception ex){ + LOG.error("fail converting alertcontext into string", ex); + return null; + } + } + + public static AlertContext fromJsonString(String json){ + ObjectMapper objectMapper = new ObjectMapper(); + try{ + Map m = objectMapper.readValue(json, Map.class); + AlertContext c = new AlertContext(); + c.addAll(m); + return c; + }catch(Exception ex){ + LOG.error("fail converting string into alertcontext", ex); + return null; + } + } + public String getProperty(String name){ return properties.get(name); } @@ -55,7 +84,7 @@ public class AlertContext implements Serializable{ public String toString(){ return properties.toString(); } - + public Map<String, String> getProperties(){ return properties; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-common/src/test/java/org/apache/eagle/common/metric/TestAlertContext.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-common/src/test/java/org/apache/eagle/common/metric/TestAlertContext.java b/eagle-core/eagle-query/eagle-common/src/test/java/org/apache/eagle/common/metric/TestAlertContext.java new file mode 100644 index 0000000..fb0c3d1 --- /dev/null +++ b/eagle-core/eagle-query/eagle-common/src/test/java/org/apache/eagle/common/metric/TestAlertContext.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.common.metric; + +import junit.framework.Assert; +import org.junit.Test; + +import java.util.HashMap; + +/** + * since 1/20/16. + */ +public class TestAlertContext { + @Test + public void test(){ + HashMap<String, String> map = new HashMap<String, String>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + AlertContext context = new AlertContext(); + context.addAll(map); + String json = context.toJsonString(); + System.out.println(json); + Assert.assertEquals(map, AlertContext.fromJsonString(json).getProperties()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinition.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinition.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinition.java index bed61af..d2d9eef 100755 --- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinition.java +++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinition.java @@ -52,6 +52,7 @@ public class EntityDefinition implements Writable{ private Map<String, Qualifier> displayNameMap = new HashMap<String, Qualifier>(); private Map<String, Qualifier> qualifierNameMap = new HashMap<String, Qualifier>(); private Map<String, Method> qualifierGetterMap = new HashMap<String, Method>(); + private String[] tags; private boolean isTimeSeries; private MetricDefinition metricDefinition; private IndexDefinition[] indexes; @@ -143,6 +144,13 @@ public class EntityDefinition implements Writable{ public void setQualifierGetterMap(Map<String, Method> qualifierGetterMap) { this.qualifierGetterMap = qualifierGetterMap; } + public String[] getTags(){ + return tags; + } + public void setTags(String[] tags){ + this.tags = tags; + } + // public Map<String,String> getQualifierDisplayNameMap(){ // Map<String,String> qualifierDisplayNameMap = new HashMap<String, String>(); // for(Map.Entry<String,Qualifier> entry: qualifierNameMap.entrySet()){ http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinitionManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinitionManager.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinitionManager.java index e144e05..d990fb5 100755 --- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinitionManager.java +++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/meta/EntityDefinitionManager.java @@ -273,6 +273,10 @@ public class EntityDefinitionManager { } } + public static void load() throws IllegalAccessException, InstantiationException { + checkInit(); + } + /** * User can register their own field SerDeser * @param clazz class of the the SerDeser @@ -424,7 +428,13 @@ public class EntityDefinitionManager { ed.setServiceCreationPath(path.path()); } } - + + final Tags tags = cls.getAnnotation(Tags.class); + if(tags != null) { + String[] tagNames = tags.value(); + ed.setTags(tagNames); + } + return ed; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/test/TestTimeSeriesAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/test/TestTimeSeriesAPIEntity.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/test/TestTimeSeriesAPIEntity.java index 1205961..6d5f560 100755 --- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/test/TestTimeSeriesAPIEntity.java +++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/test/TestTimeSeriesAPIEntity.java @@ -16,15 +16,10 @@ */ package org.apache.eagle.log.entity.test; -import org.apache.eagle.log.entity.meta.ColumnFamily; -import org.apache.eagle.log.entity.meta.Service; -import org.apache.eagle.log.entity.meta.Column; -import org.apache.eagle.log.entity.meta.TimeSeries; +import org.apache.eagle.log.entity.meta.*; import org.codehaus.jackson.map.annotate.JsonSerialize; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; -import org.apache.eagle.log.entity.meta.Prefix; -import org.apache.eagle.log.entity.meta.Table; @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @Table("unittest") @@ -32,6 +27,7 @@ import org.apache.eagle.log.entity.meta.Table; @Prefix("testTSEntity") @Service("TestTimeSeriesAPIEntity") @TimeSeries(true) +@Tags({"cluster","datacenter","random"}) public class TestTimeSeriesAPIEntity extends TaggedLogAPIEntity { @Column("a") http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/DataStorageManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/DataStorageManager.java b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/DataStorageManager.java index 23fafeb..6b22e95 100644 --- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/DataStorageManager.java +++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/DataStorageManager.java @@ -66,7 +66,7 @@ public class DataStorageManager { * * @throws IllegalDataStorageTypeException */ - public static DataStorage getDataStorageByEagleConfig(boolean cache) throws IllegalDataStorageTypeException{ + public synchronized static DataStorage getDataStorageByEagleConfig(boolean cache) throws IllegalDataStorageTypeException{ String storageType = EagleConfigFactory.load().getStorageType(); if(!cache) @@ -87,7 +87,7 @@ public class DataStorageManager { * * @throws IllegalDataStorageTypeException */ - public static DataStorage getDataStorageByEagleConfig() throws IllegalDataStorageTypeException{ + public synchronized static DataStorage getDataStorageByEagleConfig() throws IllegalDataStorageTypeException{ return getDataStorageByEagleConfig(true); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd84fa0/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java index 8f822c8..eacdcca 100644 --- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java +++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java @@ -216,6 +216,11 @@ public class CompiledQuery { this.searchCondition.setEndTime(this.rawQuery.getEndTime()); this.setStartTime(DateTimeUtil.humanDateToSeconds(this.getRawQuery().getStartTime()) * 1000); this.setEndTime(DateTimeUtil.humanDateToSeconds(this.getRawQuery().getEndTime()) * 1000); + }else{ + this.searchCondition.setStartTime("0"); + this.searchCondition.setEndTime("1"); + this.setStartTime(0); + this.setEndTime(1); } // 4. Set HBase start scanning rowkey if given
