incubator-eagle git commit: [MINOR] Fix unit test cases with ApplicationSimulator
Repository: incubator-eagle Updated Branches: refs/heads/master 12a0fb961 -> a34ef878a [MINOR] Fix unit test cases with ApplicationSimulator Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a34ef878 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a34ef878 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a34ef878 Branch: refs/heads/master Commit: a34ef878a21dada05c8c297a4b66f93889361d50 Parents: 12a0fb9 Author: Hao Chen <h...@apache.org> Authored: Mon Dec 19 23:10:26 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 19 23:13:15 2016 +0800 -- .../impl/ApplicationManagementServiceImpl.java| 2 +- .../impl/ApplicationStatusUpdateServiceImpl.java | 4 +++- .../eagle/app/test/ApplicationSimulatorImpl.java | 18 +- .../auditlog/MapRFSAuditLogAppProviderTest.java | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a34ef878/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java index 96118aa..cd97496 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java @@ -161,7 +161,7 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe return applicationEntityService.delete(appEntity); } else { -throw new ApplicationWrongStatusException("App: " + appEntity.getAppId() + " status is" + currentStatus + ", uninstall operation is not allowed"); +throw new ApplicationWrongStatusException("App: " + appEntity.getAppId() + " status is " + currentStatus + ", uninstall operation is not allowed"); } } catch (Throwable throwable) { LOGGER.error(throwable.getMessage(), throwable); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a34ef878/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationStatusUpdateServiceImpl.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationStatusUpdateServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationStatusUpdateServiceImpl.java index a1ceb83..02c3a5e 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationStatusUpdateServiceImpl.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationStatusUpdateServiceImpl.java @@ -113,7 +113,9 @@ public class ApplicationStatusUpdateServiceImpl extends ApplicationStatusUpdateS applicationEntityService.create(applicationEntity); } } -//"STOPPED" is not used in Eagle, so just do nothing. +// "STOPPED" is not used in Eagle, so just do nothing. + +applicationEntity.setStatus(topologyStatus); } catch (RuntimeException e) { LOG.error(e.getMessage(), e); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a34ef878/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/test/ApplicationSimulatorImpl.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/test/ApplicationSimulatorImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/test/ApplicationSimulatorImpl.java index 51ee2cf..1b066ef 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/test/ApplicationSimulatorImpl.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/test/ApplicationSimulatorImpl.java @@ -75,7 +75,23 @@ public class ApplicationSimulatorImpl extends ApplicationSimulator { applicationResource.startApplication(new ApplicationOperations.StartOperation(applicationEntity.getUuid())); statusUpdateService.upd
incubator-eagle git commit: [MINOR] Refine policy detail and logging format
Repository: incubator-eagle Updated Branches: refs/heads/master 725e73377 -> 12a0fb961 [MINOR] Refine policy detail and logging format Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/12a0fb96 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/12a0fb96 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/12a0fb96 Branch: refs/heads/master Commit: 12a0fb96144f70a9bd73939d7e968a2160ba Parents: 725e733 Author: Hao Chen <h...@apache.org> Authored: Mon Dec 19 20:50:17 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 19 20:50:17 2016 +0800 -- .../service/MetadataServiceClientImpl.java | 8 +- .../trigger/DynamicPolicyLoader.java| 11 +- .../app/environment/AbstractEnvironment.java| 7 +- .../environment/impl/StormExecutionRuntime.java | 8 +- .../impl/ApplicationHealthCheckServiceImpl.java | 20 +-- .../ApplicationStatusUpdateServiceImpl.java | 22 ++- .../webapp/app/dev/partials/alert/detail.html | 2 +- .../app/dev/partials/alert/policyDetail.html| 170 +-- 8 files changed, 130 insertions(+), 118 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/12a0fb96/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java index 8571e56..4565d96 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java @@ -150,7 +150,7 @@ public class MetadataServiceClientImpl implements IMetadataServiceClient { private List list(String path, GenericType<List> type) { WebResource r = client.resource(basePath + path); -LOG.info("query URL {}", basePath + path); +LOG.info("Requesting {}", basePath + path); List ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type); return ret; } @@ -172,7 +172,7 @@ public class MetadataServiceClientImpl implements IMetadataServiceClient { } private T listOne(String path, Class tClz) { -LOG.info("query URL {}", basePath + path); +LOG.info("Requesting {}", basePath + path); WebResource r = client.resource(basePath + path); ClientResponse resp = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON) @@ -181,10 +181,10 @@ public class MetadataServiceClientImpl implements IMetadataServiceClient { try { return resp.getEntity(tClz); } catch (Exception e) { -LOG.warn(" list one entity failed, ignored and continute, path {}, message {}!", path, e.getMessage()); +LOG.warn("List one entity failed, ignored and continue, path:{}", path, e); } } else { -LOG.warn("fail querying metadata service {} with http status {}", basePath + path, resp.getStatus()); +LOG.warn("Fail querying metadata service {}, http status: {}", basePath + path, resp.getStatus()); } return null; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/12a0fb96/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java index 07ae966..7d0ead5 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java @@ -55,7 +55,7 @@ public class DynamicPolicyLoader implements Runnable { // we should catch every exception to avoid zombile thread
incubator-eagle git commit: [EAGLE-843] Refactor application shared service registry framework
Repository: incubator-eagle Updated Branches: refs/heads/master a89275bfc -> 725e73377 [EAGLE-843] Refactor application shared service registry framework Refactor application shared service registry framework * Add `Optional<List> getSharedServices(Config envConfig)` in `ApplicationProvider` * Move `MRHistoryJobDailyReporter` registry to `MRHistoryJobApplicationProvider` from `ServerApplication` * Register `getSharedServices` from `ApplicationProvider` to `Environment` in `ServerApplication` * Add `EmbeddedMailService` https://issues.apache.org/jira/browse/EAGLE-843 Author: Hao Chen <h...@apache.org> Closes #750 from haoch/RefactorAppServicesFramework. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/725e7337 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/725e7337 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/725e7337 Branch: refs/heads/master Commit: 725e733778dd405c2056249e766d342cbed32e8a Parents: a89275b Author: Hao Chen <h...@apache.org> Authored: Mon Dec 19 11:16:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 19 11:16:55 2016 +0800 -- .../publisher/AlertEmailPublisherTest.java | 16 +-- .../impl/ApplicationHealthCheckServiceImpl.java | 2 +- .../eagle/app/spi/ApplicationProvider.java | 17 ++- .../queue/HadoopQueueRunningAppProvider.java| 2 +- .../MRHistoryJobApplicationProvider.java| 16 ++- .../mr/history/MRHistoryJobDailyReporter.java | 15 +- .../history/MRHistoryJobDailyReporterTest.java | 6 +- .../history/SparkHistoryJobAppProvider.java | 2 +- eagle-server/pom.xml| 6 + .../apache/eagle/server/ServerApplication.java | 39 +++-- .../eagle/server/task/ApplicationTask.java | 42 -- .../eagle/server/task/ManagedService.java | 43 ++ .../eagle/server/EmbeddedMailService.java | 141 +++ .../org/apache/eagle/server/ServerDebug.java| 7 +- .../topology/TopologyCheckAppProvider.java | 2 +- 15 files changed, 278 insertions(+), 78 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/725e7337/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/AlertEmailPublisherTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/AlertEmailPublisherTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/AlertEmailPublisherTest.java index 1f131a9..50fb07d 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/AlertEmailPublisherTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/AlertEmailPublisherTest.java @@ -38,14 +38,14 @@ public class AlertEmailPublisherTest { private SimpleSmtpServer server; @Before -public void setUp(){ +public void setUp() { config = ConfigFactory.load("application-test.conf"); server = SimpleSmtpServer.start(SMTP_PORT); } @After -public void clear(){ -if(server!=null) { +public void clear() { +if (server != null) { server.stop(); } } @@ -54,9 +54,9 @@ public class AlertEmailPublisherTest { public void testAlertEmailPublisher() throws Exception { AlertEmailPublisher publisher = new AlertEmailPublisher(); Map<String, Object> properties = new HashMap<>(); -properties.put(PublishConstants.SUBJECT,EMAIL_PUBLISHER_TEST_POLICY); -properties.put(PublishConstants.SENDER,"eagle@localhost"); -properties.put(PublishConstants.RECIPIENTS,"somebody@localhost"); +properties.put(PublishConstants.SUBJECT, EMAIL_PUBLISHER_TEST_POLICY); +properties.put(PublishConstants.SENDER, "eagle@localhost"); +properties.put(PublishConstants.RECIPIENTS, "somebody@localhost"); Publishment publishment = new Publishment(); publishment.setName("testEmailPublishment"); publishment.setType(org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher.class.getName()); @@ -65,9 +65,9 @@ public class AlertEmailPublisherTest { publishment.setSerializer(org.apache.eagle.alert.engine.publisher.impl.JsonEventSerializer.class.getName()); publishment.setProperties(properties); Map<String, String> conf = new HashMap<>(); -publisher.
incubator-eagle git commit: [EAGLE-834] Improve jpm report template by fixing font-size and notes
Repository: incubator-eagle Updated Branches: refs/heads/master 3082b5e62 -> bb71d37df [EAGLE-834] Improve jpm report template by fixing font-size and notes * Improve jpm report template by fixing font-size and notes http://issues.apache.org/jira/browse/EAGLE-834 Author: Hao Chen <h...@apache.org> Closes #746 from haoch/ImproveJPMReport. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/bb71d37d Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/bb71d37d Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/bb71d37d Branch: refs/heads/master Commit: bb71d37df856fa7738af9d654afc3a772a0ef8f8 Parents: 3082b5e Author: Hao Chen <h...@apache.org> Authored: Thu Dec 15 19:36:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Dec 15 19:36:55 2016 +0800 -- .../src/main/resources/JobReportTemplate.vm | 263 ++- .../history/MRHistoryJobDailyReporterTest.java | 6 +- 2 files changed, 88 insertions(+), 181 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/bb71d37d/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm index 424306f..c3afa58 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm @@ -76,7 +76,7 @@ } h3 { -font-size: 16px !important; +font-size: 14px !important; } .container { @@ -119,7 +119,7 @@ bgcolor="#fff"> $alert["alertTitle"] @@ -131,17 +131,19 @@ - -Job Summary - +Time: $alert["reportRange"] + -Time: $alert["reportRange"] + +Summary + @@ -164,7 +166,7 @@ valign="top"> Number of Jobs - Ratio @@ -173,13 +175,19 @@ -$item.status + #set($statusColor = "#337ab7") + #if($item.status == "SUCCEEDED") + #set($statusColor = "green") + #elseif($item.status == "FAILED" || $item.status == "KILLED") + #set($statusColor = "red") +#end +$item.status $item.numOfJobs - #set ($ratio_percentage = $item.ratio * 100) $ratio_percentage % @@ -194,12 +202,14 @@ -
incubator-eagle git commit: [MINOR] Fix bug for NPE caused by Kafka2TupleMetadata.properties and debug log checking
Repository: incubator-eagle Updated Branches: refs/heads/master 7639ff223 -> 8734c37cb [MINOR] Fix bug for NPE caused by Kafka2TupleMetadata.properties and debug log checking Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8734c37c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8734c37c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8734c37c Branch: refs/heads/master Commit: 8734c37cbaeb00120eba9127f92f7d44f0f1736e Parents: 7639ff2 Author: Hao Chen <h...@apache.org> Authored: Wed Dec 14 17:55:56 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Dec 14 17:55:56 2016 +0800 -- .../eagle/alert/coordination/model/Kafka2TupleMetadata.java | 4 +++- .../apache/eagle/app/messaging/MetricSchemaGenerator.java| 8 +--- 2 files changed, 8 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8734c37c/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java index e3ec774..eb9c18f 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java @@ -18,6 +18,8 @@ package org.apache.eagle.alert.coordination.model; import com.google.common.base.Objects; import org.apache.commons.lang3.builder.HashCodeBuilder; + +import java.util.HashMap; import java.util.Map; /** @@ -27,7 +29,7 @@ import java.util.Map; public class Kafka2TupleMetadata { private String type; private String name; // data source name -private Map<String, String> properties; +private Map<String, String> properties = new HashMap<>(); private String topic; private String schemeCls; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8734c37c/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java index 88fe338..bb29cea 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java @@ -95,7 +95,7 @@ public class MetricSchemaGenerator extends BaseRichBolt { private void createMetricSchemaEntity(String metricName, MetricDefinition metricDefinition) throws IOException, EagleServiceClientException { MetricSchemaEntity schemaEntity = new MetricSchemaEntity(); -Map<String,String> schemaTags = new HashMap<>(); +Map<String, String> schemaTags = new HashMap<>(); schemaEntity.setTags(schemaTags); schemaTags.put(MetricSchemaEntity.METRIC_NAME_TAG, metricName); schemaTags.put(MetricSchemaEntity.METRIC_TYPE_TAG, metricDefinition.getMetricType()); @@ -104,8 +104,10 @@ public class MetricSchemaGenerator extends BaseRichBolt { schemaEntity.setMetricFields(Collections.singletonList(GENERIC_METRIC_VALUE_NAME)); schemaEntity.setModifiedTimestamp(System.currentTimeMillis()); GenericServiceAPIResponseEntity response = this.client.create(Collections.singletonList(schemaEntity)); -if (response.isSuccess() && LOG.isDebugEnabled()) { -LOG.debug("Created {}", schemaEntity); +if (response.isSuccess()) { +if (LOG.isDebugEnabled()) { +LOG.debug("Created {}", schemaEntity); +} } else { LOG.error("Failed to create {}", schemaEntity, response.getException()); throw new IOException("Service error: " + response.getException());
incubator-eagle git commit: [MINOR] Modify timeout settings and support single-process mode
Repository: incubator-eagle Updated Branches: refs/heads/master 9ca2cebae -> 880ba738c [MINOR] Modify timeout settings and support single-process mode Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/880ba738 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/880ba738 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/880ba738 Branch: refs/heads/master Commit: 880ba738c54251734da14411cf873d5aebd13d76 Parents: 9ca2ceb Author: Hao Chen <h...@apache.org> Authored: Wed Dec 14 16:23:45 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Dec 14 16:23:45 2016 +0800 -- .../hadoop_jmx_collector/metric_collector.py| 32 +++- 1 file changed, 25 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/880ba738/eagle-external/hadoop_jmx_collector/metric_collector.py -- diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py index c7a5599..0f09a9e 100644 --- a/eagle-external/hadoop_jmx_collector/metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/metric_collector.py @@ -241,7 +241,7 @@ class KafkaMetricSender(MetricSender): def open(self): logging.info("Opening kafka connection for producer") -self.kafka_client = KafkaClient(self.broker_list, timeout=55) +self.kafka_client = KafkaClient(self.broker_list, timeout=50) self.kafka_producer = SimpleProducer(self.kafka_client, batch_send=False, batch_send_every_n=500, batch_send_every_t=30) self.start_time = time.time() @@ -340,18 +340,17 @@ class Runner(object): logging.exception(e) for collector in collectors: collector.join(timeout=55) -exit(0) +collector.close() except BaseException as e: if not isinstance(e, SystemExit): logging.exception(e) -exit(1) finally: for collector in collectors: if not collector.is_closed(): collector.close() @staticmethod -def run(*collectors): +def run_async(*collectors): config = None argv = sys.argv if len(argv) == 1: @@ -368,7 +367,7 @@ class Runner(object): logging.info("Starting %s", sub_process) sub_process.start() logging.info("Current PID: %s, subprocess PID: %s", current_process.pid, sub_process.pid) -sub_process.join(timeout = 55) +sub_process.join(timeout = 56) except BaseException as e: logging.exception(e) finally: @@ -378,6 +377,25 @@ class Runner(object): logging.info("%s exit code: %s", sub_process, sub_process.exitcode) exit(0) +@staticmethod +def run(*collectors): +config = None +argv = sys.argv +current_process=multiprocessing.current_process() +if len(argv) == 1: +config = Helper.load_config() +elif len(argv) == 2: +config = Helper.load_config(argv[1]) +else: +raise Exception("Usage: " + argv[0] + " CONFIG_FILE_PATH, but given too many arguments: " + str(argv)) +try: +Runner.worker(collectors, config) +except BaseException as e: +logging.exception(e) +finally: +logging.info("%s (PID: %s) exit", current_process.name, current_process.pid) +exit(0) + class JmxMetricCollector(MetricCollector): selected_domain = None listeners = [] @@ -429,13 +447,13 @@ class JmxMetricCollector(MetricCollector): reader_threads = [] for source in self.input_components: reader_thread=threading.Thread(target=self.jmx_reader, args=[source]) -reader_thread.daemon = True +reader_thread.daemon = False logging.info(reader_thread.name + " starting") reader_thread.start() reader_threads.append(reader_thread) for reader_thread in reader_threads: logging.info(reader_thread.name + " stopping") -reader_thread.join(timeout = 55) +reader_thread.join(timeout = 45) logging.info("Jmx reading threads (num: "+size+") finished")
incubator-eagle git commit: [EAGLE-839] add job diagnostics
Repository: incubator-eagle Updated Branches: refs/heads/master 87feb883c -> 84ceeb150 [EAGLE-839] add job diagnostics Author: wujinhu <wujinhu...@126.com> Closes #735 from wujinhu/EAGLE-835. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/84ceeb15 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/84ceeb15 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/84ceeb15 Branch: refs/heads/master Commit: 84ceeb150a2cdbc9a44be001fb5a86c4079644e1 Parents: 87feb88 Author: wujinhu <wujinhu...@126.com> Authored: Tue Dec 13 18:59:19 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 13 18:59:19 2016 +0800 -- .../mr/historyentity/JobExecutionAPIEntity.java | 12 +++ .../mr/history/parser/JHFEventReaderBase.java | 36 +--- .../mr/history/parser/JHFMRVer2EventReader.java | 3 ++ 3 files changed, 33 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/84ceeb15/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java -- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java index 0e40099..55233aa 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java @@ -94,7 +94,7 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity { @Column("ad") private String trackingUrl; @Column("ae") -private Map<String, Map<String, String>> failedTasks; +private String diagnostics; public String getTrackingUrl() { return trackingUrl; @@ -348,12 +348,12 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity { valueChanged("failedReduceAttempts"); } -public Map<String, Map<String, String>> getFailedTasks() { -return failedTasks; +public String getDiagnostics() { +return diagnostics; } -public void setFailedTasks(Map<String, Map<String, String>> failedTasks) { -this.failedTasks = failedTasks; -valueChanged("failedTasks"); +public void setDiagnostics(String diagnostics) { +this.diagnostics = diagnostics; +valueChanged("diagnostics"); } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/84ceeb15/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index 80cdb1c..3a9e147 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -18,6 +18,7 @@ package org.apache.eagle.jpm.mr.history.parser; +import org.apache.commons.lang3.tuple.Pair; import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig; import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter; import org.apache.eagle.jpm.mr.history.metrics.JobCounterMetricsGenerator; @@ -60,6 +61,8 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl protected Map<String, String> taskRunningHosts; // hostname to rack mapping protected Map<String, String> host2RackMapping; +// taskattempt to error msg, attemptId, taskId, error +protected Map<String, Pair<String, String>> attempt2ErrorMsg; protected String jobId; protected String jobName; @@ -105,11 +108,11 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl jobExecutionEntity.setTags(new HashMap<>(baseTags)); jobExecutionEntity.setNumFailedMaps(0); jobExecutionEntity.setNumFailedReduces(0); -jobExecutionEntity.setFailedTasks(new HashMap<>()); taskRunningHosts = new HashMap<>(); host2RackMapping = new HashMap<>(); +attempt2ErrorMsg = new HashMap<>(); taskStartTime = new Hash
incubator-eagle git commit: [EAGLE-836] Add JMX start up page
Repository: incubator-eagle Updated Branches: refs/heads/master f5f07402d -> 87feb883c [EAGLE-836] Add JMX start up page Add JMX UI application for web app usage. Author: zombieJ <smith3...@gmail.com> Closes #736 from zombieJ/EAGLE-836. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/87feb883 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/87feb883 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/87feb883 Branch: refs/heads/master Commit: 87feb883c68a9251a3e9d09222e49ea223da169d Parents: f5f0740 Author: zombieJ <smith3...@gmail.com> Authored: Tue Dec 13 18:57:22 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 13 18:57:22 2016 +0800 -- .../webapp/app/apps/hadoop_metric/.editorconfig | 27 + .../webapp/app/apps/hadoop_metric/ctrls/hdfs.js | 28 + .../app/apps/hadoop_metric/ctrls/overview.js| 72 + .../main/webapp/app/apps/hadoop_metric/index.js | 47 +++- .../apps/hadoop_metric/partials/hdfs/index.html | 19 .../apps/hadoop_metric/partials/overview.html | 49 + .../app/apps/hadoop_metric/style/index.css | 61 +++ .../hadoop_metric/widgets/availabilityChart.js | 107 +++ eagle-server/src/main/webapp/app/package.json | 2 +- 9 files changed, 410 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/87feb883/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/.editorconfig -- diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/.editorconfig b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/.editorconfig new file mode 100644 index 000..42a9b69 --- /dev/null +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/.editorconfig @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = tab +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/87feb883/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/hdfs.js -- diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/hdfs.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/hdfs.js new file mode 100644 index 000..74bb7bf --- /dev/null +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/hdfs.js @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +(function () { + /** +* `register` without params will load the module which using require +*/ + register(function (hadoopMetricApp) { + hadoopMetricApp.controller("hdfsCtrl", function ($scope, PageConfig) { + PageConfig.title = 'HDFS'; + }); + }); +})(); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/87feb883/eagle
incubator-eagle git commit: [EAGLE-838] Use subprocess for metric collector to avoid defunct process
Repository: incubator-eagle Updated Branches: refs/heads/master 7c6315311 -> f5f07402d [EAGLE-838] Use subprocess for metric collector to avoid defunct process Use subprocess for metric collector to avoid defunct process https://issues.apache.org/jira/browse/EAGLE-838 Author: Hao Chen <h...@apache.org> Closes #737 from haoch/UseSubProcessForMetricCollector. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f5f07402 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f5f07402 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f5f07402 Branch: refs/heads/master Commit: f5f07402d40c19962ea6650092509c32aafb12e6 Parents: 7c63153 Author: Hao Chen <h...@apache.org> Authored: Tue Dec 13 17:00:00 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 13 17:00:00 2016 +0800 -- .../hadoop_jmx_collector/metric_collector.py| 67 +++- 1 file changed, 37 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f5f07402/eagle-external/hadoop_jmx_collector/metric_collector.py -- diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py index 6205e1f..c7a5599 100644 --- a/eagle-external/hadoop_jmx_collector/metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/metric_collector.py @@ -26,8 +26,8 @@ import httplib import logging import threading import fnmatch -import subprocess import os +import multiprocessing # load six sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/six')) @@ -242,7 +242,7 @@ class KafkaMetricSender(MetricSender): def open(self): logging.info("Opening kafka connection for producer") self.kafka_client = KafkaClient(self.broker_list, timeout=55) -self.kafka_producer = SimpleProducer(self.kafka_client, batch_send=True, batch_send_every_n=500, +self.kafka_producer = SimpleProducer(self.kafka_client, batch_send=False, batch_send_every_n=500, batch_send_every_t=30) self.start_time = time.time() @@ -325,51 +325,58 @@ class MetricCollector(threading.Thread): class Runner(object): @staticmethod -def run(*collectors): +def worker(collectors, config): """ -Execute concurrently - -:param threads: -:return: -""" -current_pid = os.getpid() + Execute concurrently + :param threads: + :return: + """ try: -argv = sys.argv -if len(argv) == 1: -config = Helper.load_config() -elif len(argv) == 2: -config = Helper.load_config(argv[1]) -else: -raise Exception("Usage: " + argv[0] + " CONFIG_FILE_PATH, but given too many arguments: " + str(argv)) - -logging.info("PID: %s", current_pid) for collector in collectors: try: collector.init(config) collector.start() except Exception as e: logging.exception(e) - for collector in collectors: -try: -collector.join(timeout = 55) -collector.close() -except BaseException as e: -logging.exception(e) +collector.join(timeout=55) exit(0) except BaseException as e: -if isinstance(e, SystemExit): -logging.info("Exit code: %s", e) -else: +if not isinstance(e, SystemExit): logging.exception(e) exit(1) -logging.info("Exit code: 1") finally: for collector in collectors: if not collector.is_closed(): collector.close() -logging.info("Ensuring process stopped: kill -9 %s", current_pid) -subprocess.call(["kill","-9",str(current_pid)]) + +@staticmethod +def run(*collectors): +config = None +argv = sys.argv +if len(argv) == 1: +config = Helper.load_config() +elif len(argv) == 2: +config = Helper.load_config(argv[1]) +else: +raise Exception("Usage: " + argv[0] + " CONFIG_FILE_PATH, but given too many arguments: " + str(argv)) +current_process = multiprocessing.current_process() +sub_proce
incubator-eagle git commit: [EAGLE-838] Resolve defunct process in hadoop jmx script by kill -9
Repository: incubator-eagle Updated Branches: refs/heads/master b12abb38d -> b27998f82 [EAGLE-838] Resolve defunct process in hadoop jmx script by kill -9 https://issues.apache.org/jira/browse/EAGLE-838 - Resolve defunct process in hadoop jmx script by kill -9 - Support configurable log file path - Improve JMX reader to multiple-threading Author: Hao Chen <h...@apache.org> Closes #733 from haoch/FixDefunctProcess. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b27998f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b27998f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b27998f8 Branch: refs/heads/master Commit: b27998f822a492c33ed587d729ba894280660ab1 Parents: b12abb3 Author: Hao Chen <h...@apache.org> Authored: Tue Dec 13 10:41:30 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 13 10:41:30 2016 +0800 -- .../hadoop_jmx_config-sample.json | 4 +- .../hadoop_jmx_collector/hadoop_jmx_kafka.py| 2 +- .../hbase_jmx_config-sample.json| 4 +- .../hadoop_jmx_collector/metric_collector.py| 143 +-- 4 files changed, 105 insertions(+), 48 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b27998f8/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json index f5c16fd..a6ddf7d 100755 --- a/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json +++ b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json @@ -1,6 +1,8 @@ { "env": { -"site": "sandbox" +"site": "sandbox", +"metric_prefix": "hadoop.", +"log_file": "/tmp/hadoop-jmx-collector.log" }, "input": [ { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b27998f8/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py -- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py b/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py index 799d351..be7b9c7 100644 --- a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py +++ b/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py @@ -17,7 +17,7 @@ # from metric_collector import JmxMetricCollector,JmxMetricListener,Runner -import json, logging, fnmatch +import json, logging, fnmatch, sys class NNSafeModeMetric(JmxMetricListener): def on_metric(self, metric): http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b27998f8/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json b/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json index 2275ccc..c37a9ae 100644 --- a/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json +++ b/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json @@ -1,6 +1,8 @@ { "env": { -"site": "sandbox" +"site": "sandbox", +"metric_prefix": "hadoop.", +"log_file": "/tmp/hadoop-jmx-collector.log" }, "input": [ { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b27998f8/eagle-external/hadoop_jmx_collector/metric_collector.py -- diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py index c4fc457..6205e1f 100644 --- a/eagle-external/hadoop_jmx_collector/metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/metric_collector.py @@ -16,18 +16,18 @@ # limitations under the License. # -import os import re import time import json import urllib2 import sys import socket -import types import httplib import logging import threading import fnmatch +import subprocess +import os # load six sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/six')) @@ -37,11 +37,6 @@ import six sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '', 'lib/kafka-python')) from kafka import KafkaClient, SimpleProducer, SimpleConsumer -logging.basicConfig(level=logging.INFO, -format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', -date
incubator-eagle git commit: [MINOR] Fix HadoopMetricMonitorAppProviderTest for modifying app type
Repository: incubator-eagle Updated Branches: refs/heads/master 3025e5d72 -> b12abb38d [MINOR] Fix HadoopMetricMonitorAppProviderTest for modifying app type Fix HadoopMetricMonitorAppProviderTest bug for modifying app type included by #731 (EAGLE-836) Author: Hao Chen <h...@apache.org> Closes #734 from haoch/FixHadoopMetricMonitorAppBug. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b12abb38 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b12abb38 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b12abb38 Branch: refs/heads/master Commit: b12abb38d973d470f9bebdaba8e7b76b7dee5326 Parents: 3025e5d Author: Hao Chen <h...@apache.org> Authored: Mon Dec 12 20:25:02 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 12 20:25:02 2016 +0800 -- .../apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b12abb38/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java -- diff --git a/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java b/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java index eb343d9..908ce2d 100644 --- a/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java +++ b/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProviderTest.java @@ -49,11 +49,11 @@ public class HadoopMetricMonitorAppProviderTest extends ApplicationTestBase { SiteEntity siteEntity = new SiteEntity(); siteEntity.setSiteId("test_site"); siteEntity.setSiteName("Test Site"); -siteEntity.setDescription("Test Site for HADOOP_JMX_METRIC_MONITOR"); +siteEntity.setDescription("Test Site for HADOOP_METRIC_MONITOR"); siteResource.createSite(siteEntity); Assert.assertNotNull(siteEntity.getUuid()); -ApplicationOperations.InstallOperation installOperation = new ApplicationOperations.InstallOperation("test_site", "HADOOP_JMX_METRIC_MONITOR", ApplicationEntity.Mode.LOCAL); +ApplicationOperations.InstallOperation installOperation = new ApplicationOperations.InstallOperation("test_site", "HADOOP_METRIC_MONITOR", ApplicationEntity.Mode.LOCAL); installOperation.setConfiguration(getConf()); // Install application ApplicationEntity applicationEntity = applicationResource.installApplication(installOperation).getData();
incubator-eagle git commit: [EAGLE-836] Add /apps/hadoop_metric view path in HADOOP_METRIC_MONITOR app
Repository: incubator-eagle Updated Branches: refs/heads/master 268096f19 -> 3025e5d72 [EAGLE-836] Add /apps/hadoop_metric view path in HADOOP_METRIC_MONITOR app Add /apps/hadoop_metric view path in HADOOP_METRIC_MONITOR app https://issues.apache.org/jira/browse/EAGLE-836 Author: Hao Chen <h...@apache.org> Closes #731 from haoch/EAGLE-836. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3025e5d7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3025e5d7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3025e5d7 Branch: refs/heads/master Commit: 3025e5d7285ebac8236e8988df7dc52690420117 Parents: 268096f Author: Hao Chen <h...@apache.org> Authored: Mon Dec 12 15:14:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 12 15:14:55 2016 +0800 -- .../src/main/webapp/app/apps/example/index.js | 1 + eagle-hadoop-metric/pom.xml | 12 ...ache.eagle.metric.HadoopMetricMonitorAppProdiver.xml | 5 +++-- .../src/main/webapp/app/apps/hadoop_metric/index.js | 1 + eagle-hadoop-metric/src/main/webapp/app/index.html | 6 ++ 5 files changed, 23 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3025e5d7/eagle-examples/eagle-app-example/src/main/webapp/app/apps/example/index.js -- diff --git a/eagle-examples/eagle-app-example/src/main/webapp/app/apps/example/index.js b/eagle-examples/eagle-app-example/src/main/webapp/app/apps/example/index.js new file mode 100644 index 000..aab3ec7 --- /dev/null +++ b/eagle-examples/eagle-app-example/src/main/webapp/app/apps/example/index.js @@ -0,0 +1 @@ +console.log("Loading EXAMPLE_APPLICATION: /apps/example"); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3025e5d7/eagle-hadoop-metric/pom.xml -- diff --git a/eagle-hadoop-metric/pom.xml b/eagle-hadoop-metric/pom.xml index a3d1cb0..47a80e2 100644 --- a/eagle-hadoop-metric/pom.xml +++ b/eagle-hadoop-metric/pom.xml @@ -47,4 +47,16 @@ + + + + +src/main/webapp/app +assets/ + + +src/main/resources + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3025e5d7/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml -- diff --git a/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml index 87b57d6..b832b48 100644 --- a/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml +++ b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml @@ -17,9 +17,10 @@ --> -HADOOP_JMX_METRIC_MONITOR -Hadoop JMX Metric Monitor +HADOOP_METRIC_MONITOR +Hadoop Metrics Monitor 0.5.0-incubating +/apps/hadoop_metric http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3025e5d7/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/index.js -- diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/index.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/index.js new file mode 100644 index 000..6da3888 --- /dev/null +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/index.js @@ -0,0 +1 @@ +console.log("Loading HADOOP_METRIC_MONITOR: /apps/hadoop_metric"); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3025e5d7/eagle-hadoop-metric/src/main/webapp/app/index.html -- diff --git a/eagle-hadoop-metric/src/main/webapp/app/index.html b/eagle-hadoop-metric/src/main/webapp/app/index.html new file mode 100644 index 000..765d8a6 --- /dev/null +++ b/eagle-hadoop-metric/src/main/webapp/app/index.html @@ -0,0 +1,6 @@ + + +Hadoop Metric Monitor Application! +url: /apps/hadoop_metric + + \ No newline at end of file
incubator-eagle git commit: [MINOR] Create MetricSchema in batch of 1
Repository: incubator-eagle Updated Branches: refs/heads/master 7ad45fd1b -> e99b2fedf [MINOR] Create MetricSchema in batch of 1 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e99b2fed Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e99b2fed Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e99b2fed Branch: refs/heads/master Commit: e99b2fedf158f36857cc78279771e4b95a582776 Parents: 7ad45fd Author: Hao Chen <h...@apache.org> Authored: Sat Dec 10 02:29:31 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Sat Dec 10 02:29:31 2016 +0800 -- .../eagle/app/messaging/MetricSchemaGenerator.java | 15 +++ 1 file changed, 11 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e99b2fed/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java index 6563cd7..88fe338 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricSchemaGenerator.java @@ -22,10 +22,11 @@ import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Tuple; import com.typesafe.config.Config; +import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; import org.apache.eagle.metadata.model.MetricSchemaEntity; import org.apache.eagle.app.environment.builder.MetricDefinition; import org.apache.eagle.service.client.EagleServiceClientException; -import org.apache.eagle.service.client.impl.BatchSender; +import org.apache.eagle.service.client.IEagleServiceClient; import org.apache.eagle.service.client.impl.EagleServiceClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +44,7 @@ public class MetricSchemaGenerator extends BaseRichBolt { private final Config config; private OutputCollector collector; -private BatchSender client; +private IEagleServiceClient client; public MetricSchemaGenerator(MetricDefinition metricDefinition, Config config) { this.metricDefinition = metricDefinition; @@ -53,7 +54,7 @@ public class MetricSchemaGenerator extends BaseRichBolt { @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; -this.client = new EagleServiceClientImpl(config).batch(100); +this.client = new EagleServiceClientImpl(config); } @Override @@ -102,6 +103,12 @@ public class MetricSchemaGenerator extends BaseRichBolt { schemaEntity.setDimensionFields(metricDefinition.getDimensionFields()); schemaEntity.setMetricFields(Collections.singletonList(GENERIC_METRIC_VALUE_NAME)); schemaEntity.setModifiedTimestamp(System.currentTimeMillis()); -this.client.send(Collections.singletonList(schemaEntity)); +GenericServiceAPIResponseEntity response = this.client.create(Collections.singletonList(schemaEntity)); +if (response.isSuccess() && LOG.isDebugEnabled()) { +LOG.debug("Created {}", schemaEntity); +} else { +LOG.error("Failed to create {}", schemaEntity, response.getException()); +throw new IOException("Service error: " + response.getException()); +} } } \ No newline at end of file
incubator-eagle git commit: [MINOR] Fix eagle-server.sh log redirec bug
Repository: incubator-eagle Updated Branches: refs/heads/master 0fd2d7774 -> 7ad45fd1b [MINOR] Fix eagle-server.sh log redirec bug Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7ad45fd1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7ad45fd1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7ad45fd1 Branch: refs/heads/master Commit: 7ad45fd1bd63c76b294faecc7e13bad8c8145799 Parents: 0fd2d77 Author: Hao Chen <h...@apache.org> Authored: Sat Dec 10 01:20:50 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Sat Dec 10 01:20:50 2016 +0800 -- eagle-server-assembly/src/main/bin/eagle-server.sh | 10 +++--- 1 file changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7ad45fd1/eagle-server-assembly/src/main/bin/eagle-server.sh -- diff --git a/eagle-server-assembly/src/main/bin/eagle-server.sh b/eagle-server-assembly/src/main/bin/eagle-server.sh index 975d2a7..4199234 100644 --- a/eagle-server-assembly/src/main/bin/eagle-server.sh +++ b/eagle-server-assembly/src/main/bin/eagle-server.sh @@ -59,13 +59,17 @@ PROGRAM="java $JVM_OPTS -cp $EAGLE_CLASSPATH org.apache.eagle.server.ServerMain cd $DIR/../ if [ ! -e ${DIR}/../run ]; then -mkdir ${DIR}/../run; +mkdir ${DIR}/../run +fi + +if [ ! -e ${DIR}/../log ]; then +mkdir ${DIR}/../log fi start() { echo "Starting eagle service ..." echo ${PROGRAM} - nohup ${PROGRAM} 1> ${DIR}/eagle-server.out & echo $! > $PIDFILE + nohup ${PROGRAM} 1> ${DIR}/../log/eagle-server.out 2>&1 & echo $! > $PIDFILE if [ $? != 0 ];then echo "Error: failed starting" exit 1 @@ -129,4 +133,4 @@ if [ $? != 0 ]; then exit 1 fi -exit 0 \ No newline at end of file +exit 0
incubator-eagle git commit: [EAGLE-832] Support to define granularity for MetricDefinition to avoid metric duplicate
Repository: incubator-eagle Updated Branches: refs/heads/master cfd4d38fd -> 02abe02c3 [EAGLE-832] Support to define granularity for MetricDefinition to avoid metric duplicate Support to define granularity for MetricDefinition to avoid metric duplicate Author: Hao Chen <h...@apache.org> Closes #724 from haoch/SupportGranularity. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/02abe02c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/02abe02c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/02abe02c Branch: refs/heads/master Commit: 02abe02c37cd4351ea456ccf42fe24fb9544682d Parents: cfd4d38 Author: Hao Chen <h...@apache.org> Authored: Thu Dec 8 22:44:29 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Dec 8 22:44:29 2016 +0800 -- .../environment/builder/MetricDefinition.java | 21 .../app/messaging/MetricStreamPersist.java | 3 ++- .../eagle/metric/HadoopMetricMonitorApp.java| 3 +++ 3 files changed, 26 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/02abe02c/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDefinition.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDefinition.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDefinition.java index 62a81b0..d45ad2b 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDefinition.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDefinition.java @@ -19,6 +19,7 @@ package org.apache.eagle.app.environment.builder; import java.io.Serializable; import java.util.Arrays; +import java.util.Calendar; import java.util.List; import java.util.Map; @@ -40,6 +41,11 @@ public class MetricDefinition implements Serializable { private List dimensionFields; /** + * Metric granularity. + */ +private int granularity = Calendar.MINUTE; + +/** * Metric value field name. */ private String valueField = "value"; @@ -76,6 +82,13 @@ public class MetricDefinition implements Serializable { this.timestampSelector = timestampSelector; } +public int getGranularity() { +return granularity; +} + +public void setGranularity(int granularity) { +this.granularity = granularity; +} @FunctionalInterface public interface NameSelector extends Serializable { @@ -93,6 +106,14 @@ public class MetricDefinition implements Serializable { return metricDefinition; } +/** + * @see java.util.Calendar + */ +public MetricDefinition granularity(int granularity) { +this.setGranularity(granularity); +return this; +} + public static MetricDefinition namedByField(String nameField) { MetricDefinition metricDefinition = new MetricDefinition(); metricDefinition.setNameSelector(new FieldNameSelector(nameField)); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/02abe02c/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricStreamPersist.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricStreamPersist.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricStreamPersist.java index 32b168b..2a1d8a8 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricStreamPersist.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/MetricStreamPersist.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.typesafe.config.Config; import org.apache.eagle.app.environment.builder.MetricDefinition; import org.apache.eagle.app.utils.StreamConvertHelper; +import org.apache.eagle.common.DateTimeUtil; import org.apache.eagle.log.entity.GenericMetricEntity; import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; import org.apache.eagle.service.client.IEagleServiceClient; @@ -143,7 +144,7 @@ public class MetricStreamPersist extends BaseRichBolt { GenericMetricEntity entity = new GenericMetricEntity(); entity.setPrefix(metricName); -entity.setTimestamp(timestamp); + entity.setTimestamp(DateTimeUtil.roundDown(metricDefiniti
[1/2] incubator-eagle git commit: [MINOR] Assemble eagle-external/hadoop_jmx_collector to lib/scripts
Repository: incubator-eagle Updated Branches: refs/heads/master dd9cd7c51 -> cfd4d38fd [MINOR] Assemble eagle-external/hadoop_jmx_collector to lib/scripts Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c1dcfb13 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c1dcfb13 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c1dcfb13 Branch: refs/heads/master Commit: c1dcfb13b47588b886a9bc0fc002eaf87e713523 Parents: dd9cd7c Author: Hao Chen <h...@apache.org> Authored: Thu Dec 8 20:44:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Dec 8 20:44:55 2016 +0800 -- eagle-server-assembly/src/assembly/eagle-bin.xml | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c1dcfb13/eagle-server-assembly/src/assembly/eagle-bin.xml -- diff --git a/eagle-server-assembly/src/assembly/eagle-bin.xml b/eagle-server-assembly/src/assembly/eagle-bin.xml index c56b0aa..cb23c2d 100644 --- a/eagle-server-assembly/src/assembly/eagle-bin.xml +++ b/eagle-server-assembly/src/assembly/eagle-bin.xml @@ -91,5 +91,11 @@ eagle-storage-hbase-*.jar + + + + ${project.basedir}/../eagle-external/hadoop_jmx_collector +lib/scripts/hadoop_jmx_collector + - + \ No newline at end of file
incubator-eagle git commit: [EAGLE-831] UI alert support time range select
Repository: incubator-eagle Updated Branches: refs/heads/master 162aac84f -> dd9cd7c51 [EAGLE-831] UI alert support time range select * alert list support time range * policy detail page display alert template Author: zombieJ <smith3...@gmail.com> Closes #723 from zombieJ/831. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/dd9cd7c5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/dd9cd7c5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/dd9cd7c5 Branch: refs/heads/master Commit: dd9cd7c51cd5454fd201cd27377f22eb2d60d9ec Parents: 162aac8 Author: zombieJ <smith3...@gmail.com> Authored: Thu Dec 8 17:35:38 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Dec 8 17:35:38 2016 +0800 -- eagle-server/src/main/webapp/app/dev/index.html | 9 +++- .../webapp/app/dev/partials/alert/list.html | 14 +- .../app/dev/partials/alert/policyDetail.html| 47 +-- .../src/main/webapp/app/dev/public/js/app.js| 8 +++- .../webapp/app/dev/public/js/ctrls/alertCtrl.js | 48 +--- .../app/dev/public/js/services/timeSrv.js | 37 ++- 6 files changed, 104 insertions(+), 59 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/dd9cd7c5/eagle-server/src/main/webapp/app/dev/index.html -- diff --git a/eagle-server/src/main/webapp/app/dev/index.html b/eagle-server/src/main/webapp/app/dev/index.html index 93018b4..47e1b76 100644 --- a/eagle-server/src/main/webapp/app/dev/index.html +++ b/eagle-server/src/main/webapp/app/dev/index.html @@ -66,7 +66,7 @@ - + {{Time.format("startTime", Time.SHORT_FORMAT)}} ~ {{Time.format("endTime", Time.SHORT_FORMAT)}} @@ -75,6 +75,13 @@ Last 12 Hours Last 24 Hours Customize + + + + + Auto Refresh + + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/dd9cd7c5/eagle-server/src/main/webapp/app/dev/partials/alert/list.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/alert/list.html b/eagle-server/src/main/webapp/app/dev/partials/alert/list.html index 806234d..d06b41f 100644 --- a/eagle-server/src/main/webapp/app/dev/partials/alert/list.html +++ b/eagle-server/src/main/webapp/app/dev/partials/alert/list.html @@ -22,18 +22,6 @@ Alert List - - - - - - - - - - @@ -42,7 +30,7 @@ Time - + Severity Site http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/dd9cd7c5/eagl
incubator-eagle git commit: [MINOR] Refactor alert list and detail pages
Repository: incubator-eagle Updated Branches: refs/heads/master 3914e39dd -> 269ff147f [MINOR] Refactor alert list and detail pages # Changes * Refactor alert list and detail pages * Add "storage.hbase.autoCreateTable" (boolean) config to improve server startup speed with hbase Author: Hao Chen <h...@apache.org> Closes #721 from haoch/RefactorAlertPage. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/269ff147 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/269ff147 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/269ff147 Branch: refs/heads/master Commit: 269ff147f29cde959af50b1b9a4e528112ec4799 Parents: 3914e39 Author: Hao Chen <h...@apache.org> Authored: Wed Dec 7 23:28:45 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Dec 7 23:28:45 2016 +0800 -- .../apache/eagle/common/config/EagleConfig.java | 2 + .../common/config/EagleConfigConstants.java | 1 + .../eagle/common/config/EagleConfigFactory.java | 11 + .../storage/hbase/HBaseEntitySchemaManager.java | 6 ++- .../src/main/resources/application.conf | 2 + .../webapp/app/dev/partials/alert/detail.html | 46 .../webapp/app/dev/partials/alert/list.html | 35 +++ 7 files changed, 57 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/269ff147/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java -- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java index f85d161..14e2451 100755 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java @@ -57,4 +57,6 @@ public interface EagleConfig { * @return root config. */ Config getConfig(); + +boolean isAutoCreateTable(); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/269ff147/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java -- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java index c700c61..504174c 100644 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java @@ -32,6 +32,7 @@ public final class EagleConfigConstants { public static final String SERVICE_THREADPOOL_MAX_SIZE = "storage.hbase.threadpoolMaxSize"; public static final String SERVICE_THREADPOOL_SHRINK_SIZE = "storage.hbase.threadpoolShrinkSize"; public static final String SERVICE_AUDITING_ENABLED = "storage.hbase.auditEnabled"; +public static final String AUTO_CREATE_TABLE = "storage.hbase.autoCreateTable"; public static final String EAGLE_TIME_ZONE = "service.timezone"; public static final String DEFAULT_EAGLE_TIME_ZONE = "UTC"; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/269ff147/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java -- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java index 65d93e9..e95e65c 100755 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java @@ -43,6 +43,7 @@ public class EagleConfigFactory implements EagleConfig { private String storageType; private Config config; private TimeZone timeZone; +private boolean autoCreateTable = true; public boolean isCoprocessorEnabled() { return isCoprocessorEnabled; @@ -102,6 +103,7 @@ public class EagleConfigFactory implements EagleConfig { final String zkZnodeParent = config.hasPath(EagleConfigConstants.SERVICE_ZOOKEEPER_ZNODE_PARENT) ? config.getString(EagleConfigConstants.SERVICE_ZOOKEEPER_ZNODE_PARENT) : EagleConfigCon
incubator-eagle git commit: [MINOR] Rename beam_group_filter to bean_group_filter
Repository: incubator-eagle Updated Branches: refs/heads/master 8416f64da -> 5ef6c183f [MINOR] Rename beam_group_filter to bean_group_filter Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/5ef6c183 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/5ef6c183 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/5ef6c183 Branch: refs/heads/master Commit: 5ef6c183f5b8708f83d08c5e51b2556c6371b82a Parents: 8416f64 Author: Hao Chen <h...@apache.org> Authored: Wed Dec 7 10:57:12 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Dec 7 10:57:12 2016 +0800 -- .../hadoop_jmx_collector/hadoop_jmx_config-sample.json | 4 ++-- eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json | 2 +- eagle-external/hadoop_jmx_collector/metric_collector.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5ef6c183/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json old mode 100644 new mode 100755 index 07bdc84..f5c16fd --- a/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json +++ b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json @@ -23,7 +23,7 @@ } ], "filter": { -"beam_group_filter": ["hadoop","java.lang"], +"bean_group_filter": ["hadoop","java.lang"], "metric_name_filter": [ "hadoop.memory.heapmemoryusage.used", "hadoop.memory.nonheapmemoryusage.used", @@ -67,4 +67,4 @@ ] } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5ef6c183/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json b/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json index 906f134..2275ccc 100644 --- a/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json +++ b/eagle-external/hadoop_jmx_collector/hbase_jmx_config-sample.json @@ -17,7 +17,7 @@ } ], "filter": { -"beam_group_filter": ["hadoop","java.lang","java.nio"], +"bean_group_filter": ["hadoop","java.lang","java.nio"], "metric_name_filter": [ "hadoop.memory.heapmemoryusage.used", "hadoop.memory.nonheapmemoryusage.used", http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5ef6c183/eagle-external/hadoop_jmx_collector/metric_collector.py -- diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py index 4b7b209..c4fc457 100644 --- a/eagle-external/hadoop_jmx_collector/metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/metric_collector.py @@ -359,7 +359,7 @@ class JmxMetricCollector(MetricCollector): raise Exception("port not defined in " + str(input)) if not input.has_key("https"): input["https"] = False -self.selected_domain = [s.encode('utf-8') for s in config[u'filter'].get('beam_group_filter')] +self.selected_domain = [s.encode('utf-8') for s in config[u'filter'].get('bean_group_filter')] if config["env"].has_key("metric_prefix"): self.metric_prefix = config["env"]["metric_prefix"] logging.info("Override env.metric_prefix: " + self.metric_prefix + ", default: hadoop.")
incubator-eagle git commit: [EAGLE-825] Improve jmx collector with built-in metric filter
Repository: incubator-eagle Updated Branches: refs/heads/master 188ddf500 -> 8416f64da [EAGLE-825] Improve jmx collector with built-in metric filter ## Changes * Support wildcard metric name filter * Add built-in important metric name filter for hadoop and hbase ## JIRA https://issues.apache.org/jira/browse/EAGLE-825 Author: Hao Chen <h...@apache.org> Closes #715 from haoch/ImproveJMXCollector. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8416f64d Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8416f64d Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8416f64d Branch: refs/heads/master Commit: 8416f64da7dcba0c48527aa678a5b2ec9a37d4a8 Parents: 188ddf5 Author: Hao Chen <h...@apache.org> Authored: Tue Dec 6 19:47:50 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 6 19:47:50 2016 +0800 -- .../hadoop_jmx_collector/config-sample.json | 46 - .../hadoop_jmx_config-sample.json | 70 + .../hadoop_jmx_collector/hadoop_jmx_kafka.py| 15 ++- .../hbase_jmx_config-sample.json| 94 + .../hadoop_jmx_collector/metric_collector.py| 101 --- 5 files changed, 265 insertions(+), 61 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8416f64d/eagle-external/hadoop_jmx_collector/config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/config-sample.json b/eagle-external/hadoop_jmx_collector/config-sample.json deleted file mode 100644 index a0670e1..000 --- a/eagle-external/hadoop_jmx_collector/config-sample.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "env": { -"site": "sandbox" - }, - "input": [ -{ - "component": "namenode", - "host": "sandbox.hortonworks.com", - "port": "50070", - "https": false -}, -{ - "component": "namenode", - "host": "sandbox.hortonworks.com", - "port": "50070", - "https": false -}, -{ - "component": "resourcemanager", - "host": "sandbox.hortonworks.com", - "port": "8088", - "https": false -}, -{ - "component": "resourcemanager", - "host": "sandbox.hortonworks.com", - "port": "8088", - "https": false -} - ], - "filter": { -"monitoring.group.selected": ["hadoop", "java.lang"] - }, - "output": { -"kafka": { - "default_topic": "nn_jmx_metric_sandbox", - "component_topic_mapping": { -"namenode": "nn_jmx_metric_sandbox", -"resourcemanager": "rm_jmx_metric_sandbox" - }, - "broker_list": [ -"sandbox.hortonworks.com:6667" - ] -} - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8416f64d/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json -- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json new file mode 100644 index 000..07bdc84 --- /dev/null +++ b/eagle-external/hadoop_jmx_collector/hadoop_jmx_config-sample.json @@ -0,0 +1,70 @@ +{ + "env": { +"site": "sandbox" + }, + "input": [ +{ + "component": "namenode", + "host": "sandbox.hortonworks.com", + "port": "50070", + "https": false +}, +{ + "component": "resourcemanager", + "host": "sandbox.hortonworks.com", + "port": "19888", + "https": false +}, +{ + "component": "datanode", + "host": "sandbox.hortonworks.com", + "port": "50075", + "https": false +} + ], + "filter": { +"beam_group_filter": ["hadoop","java.lang"], +"metric_name_filter": [ + "hadoop.memory.heapmemoryusage.used", + "hadoop.memory.nonheapmemoryusage.used", + "hadoop
incubator-eagle git commit: [MINOR] Fix initial sql syntax problem
Repository: incubator-eagle Updated Branches: refs/heads/master 5da9df822 -> 188ddf500 [MINOR] Fix initial sql syntax problem [MINOR]Fix initial sql syntax problem - Fix sql comment. - Modify table name to lowercase, or got error "ERROR [2016-12-06 08:16:15,738] org.apache.eagle.alert.metadata.impl.JdbcMetadataHandler: Table 'eagle.kafka_tuple_metadata' doesn't exist". Author: chitin <chitin1...@gmail.com> Closes #716 from chitin/MINOR_SQL. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/188ddf50 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/188ddf50 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/188ddf50 Branch: refs/heads/master Commit: 188ddf50086ef29b4af5c6afe498197dbc7aa0aa Parents: 5da9df8 Author: chitin <chitin1...@gmail.com> Authored: Tue Dec 6 19:14:18 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Dec 6 19:14:18 2016 +0800 -- eagle-server-assembly/src/main/bin/createTables.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/188ddf50/eagle-server-assembly/src/main/bin/createTables.sql -- diff --git a/eagle-server-assembly/src/main/bin/createTables.sql b/eagle-server-assembly/src/main/bin/createTables.sql index c5472b6..b6e635a 100644 --- a/eagle-server-assembly/src/main/bin/createTables.sql +++ b/eagle-server-assembly/src/main/bin/createTables.sql @@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS hbase_sensitivity_entity ( primary key (site, hbase_resource) ); alert engine metadata +-- alert engine metadata CREATE TABLE IF NOT EXISTS stream_cluster ( id VARCHAR (50) PRIMARY KEY, @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS stream_definition ( content longtext DEFAULT NULL ); -CREATE TABLE IF NOT EXISTS Kafka_tuple_metadata ( +CREATE TABLE IF NOT EXISTS kafka_tuple_metadata ( id VARCHAR (50) PRIMARY KEY, content longtext DEFAULT NULL );
[3/3] incubator-eagle git commit: [EAGLE-815] Add eagle alert template, severity and category support
[EAGLE-815] Add eagle alert template, severity and category support Support alert template to generate human readable message # New Features * Support to define alert template in PolicyDefinition * Support to generate alert event (subject, instead of using AlertPublishEvent for less changes) based on policy'template and alert event as context * Dynamically load policy metadata in alert publisher * Integration VelocityAlertTemplateEngine in alertPublisherBolt with metadata lifecycle * Support persist alert message in AlertEntity as immutable field. * Refactor alert mail template using https://github.com/mailgun/transactional-email-templates * Refactor Alert Template to become simple and human-readable. * Add Alert Category * Add Alert Severity (notification template color will change according to different severiy): UNKNOWN (blue), OK(green), WARNING (orange), CRITICAL/FATAL (dark black) * Add Alert Subject Template Engine, for example: * Sample Event: { "host": "localhost", "timestamp": 1480319108000, "metric": "hadoop.cpu.usage", "component": "namenode", "site": "test2", "value": 0.96 } * Sample Subject âRESOURCEMANAGER JMX Metric Alertâ is defined by: $component.toUpperCase() JMX Metric Alert * Sample Body is defined as: An alert happened on $component ($host) of cluster $site at $ALERT_TIME because $metric = $value * Add VelocityTemplateParser # TODO * Integrate velocity parser into policy validator, make sure the template only contains `global variable` and `alert stream schema fields`. * Add alert definition preview Author: Hao Chen <h...@apache.org> Closes #711 from haoch/EAGLE-815. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/1c81c086 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/1c81c086 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/1c81c086 Branch: refs/heads/master Commit: 1c81c0865c70b9a0a91b86d7813be923e86e1dba Parents: e5e215e Author: Hao Chen <h...@apache.org> Authored: Mon Dec 5 14:12:49 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Dec 5 14:12:49 2016 +0800 -- .../eagle/alert/app/AlertEagleStorePlugin.java | 14 +- .../engine/coordinator/AlertDefinition.java | 72 +++ .../alert/engine/coordinator/AlertSeverity.java | 21 + .../engine/coordinator/PolicyDefinition.java| 18 + .../alert/engine/model/AlertPublishEvent.java | 81 ++- .../alert/engine/model/AlertStreamEvent.java| 60 ++- .../engine/model/AlertPublishEventTest.java | 2 +- .../engine/model/AlertStreamEventTest.java | 17 +- .../interpreter/PolicyExecutionPlanner.java | 3 + .../engine/publisher/AlertStreamFilter.java | 26 + .../engine/publisher/PipeStreamFilter.java | 46 ++ .../engine/publisher/PublishConstants.java | 7 +- .../publisher/email/AlertEmailGenerator.java| 56 ++- .../publisher/impl/AlertEmailPublisher.java | 4 +- .../publisher/template/AlertContextFields.java | 43 ++ .../publisher/template/AlertTemplateEngine.java | 48 ++ .../template/AlertTemplateProvider.java | 23 + .../template/VelocityAlertTemplateEngine.java | 170 +++ .../template/VelocityTemplateParser.java| 95 .../alert/engine/runner/AlertPublisherBolt.java | 128 +++-- .../src/main/resources/ALERT_DEFAULT.vm | 301 --- .../main/resources/ALERT_DEFAULT_TEMPLATE.vm| 301 +++ .../main/resources/ALERT_INLINED_TEMPLATE.vm| 259 ++ .../src/main/resources/ALERT_LIGHT_TEMPLATE.vm | 495 +++ .../publisher/AlertPublisherTestHelper.java | 19 +- .../dedup/DefaultDedupWithoutStateTest.java | 2 +- .../dedup/DefaultDeduplicatorTest.java | 2 +- .../VelocityAlertTemplateEngineTest.java| 135 + .../template/VelocityTemplateParserTest.java| 65 +++ .../template/VelocityTemplateTest.java | 95 .../app/messaging/KafkaStreamProvider.java | 35 +- .../eagle/app/messaging/KafkaStreamSource.java | 7 +- .../app/messaging/KafkaStreamSourceConfig.java | 4 +- .../eagle/metadata/model/AlertEntity.java | 21 + ...le.metric.HadoopMetricMonitorAppProdiver.xml | 50 +- .../eagle/metric/SendSampleDataToKafka.java | 15 +- .../resources/hadoop_jmx_metric_sample.json | 2 +- .../test/resources/integrate_test_policy.json | 37 ++ .../history/storm/SparkHistoryJobParseBolt.java | 2 +- .../partials/alert/policyEdit/advancedMode.html | 25 +- .../app/dev/public/js/ctrls/alertEditCtrl.js| 6 + 41 files changed, 2386 insertions(+), 426 de
[2/3] incubator-eagle git commit: [EAGLE-815] Add eagle alert template, severity and category support
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1c81c086/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_DEFAULT_TEMPLATE.vm -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_DEFAULT_TEMPLATE.vm b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_DEFAULT_TEMPLATE.vm new file mode 100644 index 000..3926cc8 --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_DEFAULT_TEMPLATE.vm @@ -0,0 +1,301 @@ + +http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd;> +http://www.w3.org/1999/xhtml;> +#set ( $alert = $alertList[0] ) + + + +$alert["alertSubject"] + +body { +width: 100% !important; +min-width: 100%; +-webkit-text-size-adjust: 100%; +-ms-text-size-adjust: 100%; +margin: 0; +padding: 0; +} + +table { +border-spacing: 0; +border-collapse: collapse; +} + +table th, +table td { +padding: 3px 0 3px 0; +} + +.body { +width: 100%; +} + +p, a, h1, h2, h3, ul, ol, li { +font-family: Helvetica, Arial, sans-serif; +font-weight: normal; +margin: 0; +padding: 0; +} + +p { +font-size: 14px; +line-height: 19px; +} + +a { +color: #3294b1; +} + +h1 { +font-size: 36px; +margin: 15px 0 5px 0; +} + +h2 { +font-size: 32px; +} + +h3 { +font-size: 28px; +} + +ul, ol { +margin: 0 0 0 25px; +padding: 0; +} + +.btn { +background: #2ba6cb !important; +border: 1px solid #2284a1; +padding: 10px 20px 10px 20px; +text-align: center; +} + +.btn:hover { +background: #2795b6 !important; +} + +.btn a { +color: #FF; +text-decoration: none; +font-weight: bold; +padding: 10px 20px 10px 20px; +} + +.tableBordered { +border-top: 1px solid #b9e5ff; +} + +.tableBordered th { +background: #ECF8FF; +} + +.tableBordered th p { +font-weight: bold; +color: #3294b1; +} + +.tableBordered th, +.tableBordered td { +color: #33; +border-bottom: 1px solid #b9e5ff; +text-align: center; +padding-bottom: 5px; +} + +.panel { +height: 100px; +} + + + + + + + + + + +Eagle Alert Notification + + + + + + + + + + + + +Warning: $alert["alertSubject"] + + + + + + + + +Detected Time: $alert["alertTime"] + +#set ( $alertSeverity = $alert["alertSeverity"] ) +#if (!$alertSeverity || ("$alertSeverity" == "")) +#set ( $alert["alertSeverity"] = "WARNING") +#end + + +Severity: +#if ($alert["alertSeverity"] == "WARNING") + $alert["alertSeverity"] +#else +$alert["alertSeverity"] +#end + + + + + + + + + + +Alert Message + + + + + +$alert["alertBody"] + + + + + +Alert Detail + + + + + + +
[1/3] incubator-eagle git commit: [EAGLE-815] Add eagle alert template, severity and category support
Repository: incubator-eagle Updated Branches: refs/heads/master e5e215e0b -> 1c81c0865 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1c81c086/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/SendSampleDataToKafka.java -- diff --git a/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/SendSampleDataToKafka.java b/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/SendSampleDataToKafka.java index 67b94f8..026f60b 100644 --- a/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/SendSampleDataToKafka.java +++ b/eagle-hadoop-metric/src/test/java/org/apache/eagle/metric/SendSampleDataToKafka.java @@ -16,21 +16,25 @@ */ package org.apache.eagle.metric; -import com.google.common.base.Preconditions; import com.typesafe.config.ConfigFactory; import kafka.producer.KeyedMessage; import kafka.producer.ProducerConfig; -import org.apache.commons.io.IOUtils; import org.apache.eagle.app.messaging.KafkaStreamProvider; import org.apache.eagle.app.messaging.KafkaStreamSinkConfig; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; import java.util.Properties; public class SendSampleDataToKafka { public static void main(String[] args) throws URISyntaxException, IOException { +String data = "{" + +"\"host\":\"localhost\", " + +"\"timestamp\": 1480319109000, " + +"\"metric\": \"hadoop.cpu.usage\", " + +"\"component\": \"namenode\", " + +"\"site\": \"test\", " + +"\"value\": 0.98}"; KafkaStreamSinkConfig config = new KafkaStreamProvider().getSinkConfig("HADOOP_JMX_METRIC_STREAM",ConfigFactory.load()); Properties properties = new Properties(); properties.put("metadata.broker.list", config.getBrokerList()); @@ -44,10 +48,7 @@ public class SendSampleDataToKafka { ProducerConfig producerConfig = new ProducerConfig(properties); kafka.javaapi.producer.Producer producer = new kafka.javaapi.producer.Producer(producerConfig); try { -InputStream is = SendSampleDataToKafka.class.getResourceAsStream("hadoop_jmx_metric_sample.json"); -Preconditions.checkNotNull(is, "hadoop_jmx_metric_sample.json"); -String value = IOUtils.toString(is); -producer.send(new KeyedMessage(config.getTopicId(), value)); +producer.send(new KeyedMessage(config.getTopicId(), data)); } finally { producer.close(); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1c81c086/eagle-hadoop-metric/src/test/resources/hadoop_jmx_metric_sample.json -- diff --git a/eagle-hadoop-metric/src/test/resources/hadoop_jmx_metric_sample.json b/eagle-hadoop-metric/src/test/resources/hadoop_jmx_metric_sample.json index f0f62f2..68472a0 100644 --- a/eagle-hadoop-metric/src/test/resources/hadoop_jmx_metric_sample.json +++ b/eagle-hadoop-metric/src/test/resources/hadoop_jmx_metric_sample.json @@ -1,6 +1,6 @@ { "host":"localhost", - "timestamp": 1480319107, + "timestamp": 1480319107000, "metric": "hadoop.cpu.usage", "component": "namenode", "site": "test", http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1c81c086/eagle-hadoop-metric/src/test/resources/integrate_test_policy.json -- diff --git a/eagle-hadoop-metric/src/test/resources/integrate_test_policy.json b/eagle-hadoop-metric/src/test/resources/integrate_test_policy.json new file mode 100644 index 000..e9de11c --- /dev/null +++ b/eagle-hadoop-metric/src/test/resources/integrate_test_policy.json @@ -0,0 +1,37 @@ +{ + "name": "TEST_POLICY", + "description": "from HADOOP_JMX_METRIC_STREAM_SANDBOX[site==\"test\"] insert into ALERT_STREAM;", + "inputStreams": [ +"HADOOP_JMX_METRIC_STREAM_SANDBOX" + ], + "outputStreams": [ +"ALERT_STREAM" + ], + "definition": { +"type": "siddhi", +"value": "from HADOOP_JMX_METRIC_STREAM_SANDBOX[site==\"test\"] select component,value, name insert into ALERT_STREAM;", +"handlerClass": null, +"properties": {}, +"inputStreams": [], +"outputStreams": [] + }, + "stateDefinition": null, + "policyStatus": "ENABLED", + "alertDefinition": { +"templateType": "TEXT", +"subject": "$component Disk Usage $value", +"body": "#set($usage_percentage = $value * 100)\r\n\r\nDisk Usage ($metric) reached $usage_percentage % (Threshold: 90%) on $component of cluster: $site on:\r\n\r\nHostname: $host\r\nWhen: $ALERT_TIME\r\n Root Cause: UNKNOWN\r\nAction Required: N/A\r\n", +"severity": "CRITICAL", +"category": "HDFS" + }, + "partitionSpec": [ +{ + "streamId": "HADOOP_JMX_METRIC_STREAM_SANDBOX", + "type": "SHUFFLE", + "columns": [], +
incubator-eagle git commit: [EAGLE-816] Fix host problem in jmx collector script
Repository: incubator-eagle Updated Branches: refs/heads/master f39ffe915 -> 68a3e8bde [EAGLE-816] Fix host problem in jmx collector script [EAGLE-816] Fix host problem in jmx collector script Fix host name bug in jmx collector scripts, which current always only use current fadn instead of configured host, it's a bug. Author: Hao Chen <h...@apache.org> Closes #701 from haoch/EAGLE-816. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/68a3e8bd Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/68a3e8bd Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/68a3e8bd Branch: refs/heads/master Commit: 68a3e8bde3418add34c0b585a206236ee227c6e0 Parents: f39ffe9 Author: Hao Chen <h...@apache.org> Authored: Wed Nov 30 18:00:51 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 30 18:00:51 2016 +0800 -- .../hadoop_jmx_collector/hadoop_ha_checker.py | 19 +-- .../hadoop_jmx_collector/metric_collector.py| 20 ++-- 2 files changed, 27 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/68a3e8bd/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py -- diff --git a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py index e8bf4ae..dcca28e 100644 --- a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py +++ b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py @@ -17,17 +17,19 @@ # from metric_collector import MetricCollector, JmxReader, YarnWSReader, Runner -import logging,socket +import logging,socket,string class HadoopNNHAChecker(MetricCollector): def run(self): hosts = [] - +host_name_list = [] for input in self.config["input"]: if not input.has_key("host"): input["host"] = socket.getfqdn() if input.has_key("component") and input["component"] == "namenode": hosts.append(input) +host_name_list.append(input["host"]) + if not bool(hosts): logging.warn("non hosts are configured as 'namenode' in 'input' config, exit") return @@ -35,7 +37,10 @@ class HadoopNNHAChecker(MetricCollector): logging.info("Checking namenode HA: " + str(hosts)) total_count = len(hosts) +all_hosts_name = string.join(host_name_list,",") + self.collect({ +"host": all_hosts_name, "component": "namenode", "metric": "hadoop.namenode.hastate.total.count", "value": total_count @@ -63,18 +68,21 @@ class HadoopNNHAChecker(MetricCollector): logging.exception("failed to read jmx from " + host["host"] + ":" + host["port"]) failed_count += 1 self.collect({ +"host": all_hosts_name, "component": "namenode", "metric": "hadoop.namenode.hastate.active.count", "value": active_count }) self.collect({ +"host": all_hosts_name, "component": "namenode", "metric": "hadoop.namenode.hastate.standby.count", "value": standby_count }) self.collect({ +"host": all_hosts_name, "component": "namenode", "metric": "hadoop.namenode.hastate.failed.count", "value": failed_count @@ -83,19 +91,23 @@ class HadoopNNHAChecker(MetricCollector): class HadoopRMHAChecker(MetricCollector): def run(self): hosts = [] +all_hosts = [] for input in self.config["input"]: if not input.has_key("host"): input["host"] = socket.getfqdn() if input.has_key("component") and input["component"] == "resourcemanager": hosts.append(input) +all_hosts.append(input["host"]) if not bool(hosts): logging.warn("Non hosts are configured as 'resourcemanager' in 'input' config, exit") return logging.info("Checking resource manager HA: " + str(hosts)) total_count = len(hosts)
incubator-eagle git commit: [MINOR] Access resource manager REST API anonymously
Repository: incubator-eagle Updated Branches: refs/heads/master 10572c296 -> e7a80deb0 [MINOR] Access resource manager REST API anonymously Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e7a80deb Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e7a80deb Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e7a80deb Branch: refs/heads/master Commit: e7a80deb051cdd15a87a13ec7e1956a4704d53a0 Parents: 10572c2 Author: Hao Chen <h...@apache.org> Authored: Wed Nov 30 12:26:22 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 30 12:26:22 2016 +0800 -- eagle-external/hadoop_jmx_collector/metric_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e7a80deb/eagle-external/hadoop_jmx_collector/metric_collector.py -- diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py index 45a3877..9a4a618 100644 --- a/eagle-external/hadoop_jmx_collector/metric_collector.py +++ b/eagle-external/hadoop_jmx_collector/metric_collector.py @@ -164,7 +164,7 @@ class YarnWSReader: self.https = https def read_cluster_info(self): -cluster_info = Helper.http_get(self.host, self.port, self.https, "/ws/v1/cluster/info") +cluster_info = Helper.http_get(self.host, self.port, self.https, "/ws/v1/cluster/info?anonymous=true") logging.debug(cluster_info) return json.loads(cluster_info)
[4/5] incubator-eagle git commit: [EAGLE-806] Integrate Metric Process and Persistence with Application Framework
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSinkConfig.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSinkConfig.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSinkConfig.java new file mode 100644 index 000..4a72709 --- /dev/null +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSinkConfig.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.app.messaging; + +import org.apache.eagle.metadata.model.StreamSinkConfig; + +/** + * FIXME Rename to KafkaStreamMessagingConfig. + */ +public class KafkaStreamSinkConfig implements StreamSinkConfig { +// Write Config +private String topicId; +private String brokerList; +private String serializerClass; +private String keySerializerClass; +private String numBatchMessages; +private String maxQueueBufferMs; +private String producerType; +private String requestRequiredAcks; + +public String getTopicId() { +return topicId; +} + +public void setTopicId(String topicId) { +this.topicId = topicId; +} + +public String getBrokerList() { +return brokerList; +} + +public void setBrokerList(String brokerList) { +this.brokerList = brokerList; +} + +public String getSerializerClass() { +return serializerClass; +} + +public void setSerializerClass(String serializerClass) { +this.serializerClass = serializerClass; +} + +public String getKeySerializerClass() { +return keySerializerClass; +} + +public void setKeySerializerClass(String keySerializerClass) { +this.keySerializerClass = keySerializerClass; +} + +public String getNumBatchMessages() { +return numBatchMessages; +} + +public void setNumBatchMessages(String numBatchMessages) { +this.numBatchMessages = numBatchMessages; +} + +public String getMaxQueueBufferMs() { +return maxQueueBufferMs; +} + +public void setMaxQueueBufferMs(String maxQueueBufferMs) { +this.maxQueueBufferMs = maxQueueBufferMs; +} + +public String getProducerType() { +return producerType; +} + +public void setProducerType(String producerType) { +this.producerType = producerType; +} + +public String getRequestRequiredAcks() { +return requestRequiredAcks; +} + +public void setRequestRequiredAcks(String requestRequiredAcks) { +this.requestRequiredAcks = requestRequiredAcks; +} + +@Override +public String getType() { +return "KAFKA"; +} + +@Override +public Class getSinkType() { +return KafkaStreamSink.class; +} + +@Override +public Class getConfigType() { +return KafkaStreamSinkConfig.class; +} + + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSource.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSource.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSource.java new file mode 100644 index 000..9edf644 --- /dev/null +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/messaging/KafkaStreamSource.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * +
[3/5] incubator-eagle git commit: [EAGLE-806] Integrate Metric Process and Persistence with Application Framework
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java index 7495ef5..40f08c1 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java @@ -45,7 +45,7 @@ public class JDBCDataSourceProvider implements Provider { @Override public void run() { try { -LOGGER.info("Shutting down data source"); +LOGGER.info("Shutting down data fromStream"); datasource.close(); } catch (SQLException e) { LOGGER.error("SQLException: {}", e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java index 6ddf3c6..02bf1cc 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java @@ -210,10 +210,10 @@ public class ApplicationEntityServiceJDBCImpl implements ApplicationEntityServic ExecutionRuntime runtime = ExecutionRuntimeManager.getInstance().getRuntime( applicationProviderService.getApplicationProviderByType(entity.getDescriptor().getType()).getApplication().getEnvironmentType(), config); StreamSinkConfig streamSinkConfig = runtime.environment() - .streamSink().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(), copied.getStreamId()), effectiveConfig); + .stream().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(), copied.getStreamId()), effectiveConfig); StreamDesc streamDesc = new StreamDesc(); streamDesc.setSchema(copied); -streamDesc.setSink(streamSinkConfig); +streamDesc.setSinkConfig(streamSinkConfig); streamDesc.setStreamId(copied.getStreamId()); streamDesc.getSchema().setDataSource(entity.getAppId()); return streamDesc; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java -- diff --git a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java index d06a3e4..fa59e8a 100644 --- a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java +++ b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java @@ -25,7 +25,8 @@ import java.io.Serializable; /** * Some common codes to enable DAO through eagle service including service host/post, credential population etc. */ -public class EagleServiceConnector implements Serializable{ +@Deprecated +public class EagleServiceConnector implements Serializable { private final String eagleServiceHost; private final Integer eagleServicePort; private String username; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/impl/EagleServiceClientImpl.java -- diff --git a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/impl/EagleServiceClientImpl.java
[2/5] incubator-eagle git commit: [EAGLE-806] Integrate Metric Process and Persistence with Application Framework
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/10572c29/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java -- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java index f3eac23..94b3727 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java @@ -1,165 +1,165 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.topology.extractor.hbase; - -import org.apache.eagle.app.utils.HadoopSecurityUtil; -import org.apache.eagle.topology.TopologyCheckAppConfig; -import org.apache.eagle.topology.TopologyConstants; -import org.apache.eagle.topology.extractor.TopologyEntityParserResult; -import org.apache.eagle.topology.entity.HBaseServiceTopologyAPIEntity; -import org.apache.eagle.topology.extractor.TopologyEntityParser; -import org.apache.eagle.topology.resolver.TopologyRackResolver; -import org.apache.eagle.topology.utils.EntityBuilderHelper; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.ServerLoad; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.HBaseAdmin; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.eagle.topology.TopologyConstants.*; - -public class HbaseTopologyEntityParser implements TopologyEntityParser { - -private Configuration hBaseConfiguration; -private String site; -private Boolean kerberosEnable = false; -private TopologyRackResolver rackResolver; - -public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) { -this.site = site; -this.rackResolver = resolver; -this.hBaseConfiguration = HBaseConfiguration.create(); -this.hBaseConfiguration.set("hbase.zookeeper.quorum", hBaseConfig.zkQuorum); -this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", hBaseConfig.zkClientPort); -this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot); -this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes); -// kerberos authentication -if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null -&& !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) { - this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal); - this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab); -this.kerberosEnable = true; -this.hBaseConfiguration.set("hbase.security.authentication", "kerberos"); -this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal); -} -} - -private HBaseAdmin getHBaseAdmin() throws IOException { -if (this.kerberosEnable) { -HadoopSecurityUtil.login(hBaseConfiguration); -} -return new HBaseAdmin(this.hBaseConfiguration); -} - -@Override -public TopologyEntityParserResult parse(long timestamp) throws IOException { -long deadServers = 0; -long liveServers = 0; -TopologyEntityParserResult result = new TopologyEntityParserResult(); -HBaseAdmin admin = null; -try { -admin = getHBaseAdmin(); -ClusterStatus status = admin.getClusterStatus(); -deadServers = status.getDeadServers(); -liveServers = status.getServersSize(); -
[5/6] incubator-eagle git commit: [MINOR] Sync remote to resolve bug caused by EAGLE-791
[MINOR] Sync remote to resolve bug caused by EAGLE-791 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/229d6b32 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/229d6b32 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/229d6b32 Branch: refs/heads/master Commit: 229d6b32f8e4d5e87c623e58ca9ac0a44346ff2a Parents: 225d539 dc9fe0b Author: Hao Chen <h...@apache.org> Authored: Tue Nov 22 15:16:37 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Nov 22 15:16:37 2016 +0800 -- --
[2/6] incubator-eagle git commit: [EAGLE-427] support timestamp in a query statement
[EAGLE-427] support timestamp in a query statement https://issues.apache.org/jira/browse/EAGLE-427 Author: Zhao, Qingwen <qingwz...@apache.org> Closes #672 from qingwen220/EAGLE-427. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/225d5397 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/225d5397 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/225d5397 Branch: refs/heads/master Commit: 225d53973f4016e9fb22cf95b3166b2ef27e7a8d Parents: 7471ece Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Tue Nov 22 14:27:32 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Nov 22 15:15:08 2016 +0800 -- .../org/apache/eagle/common/DateTimeUtil.java | 8 ++ .../entity/GenericEntityScanStreamReader.java | 4 +-- .../log/entity/GenericEntityStreamReaderMT.java | 12 - ...ricMetricEntityDecompactionStreamReader.java | 4 +-- .../eagle/log/entity/SearchCondition.java | 12 - .../apache/eagle/query/TestHBaseLogReader2.java | 28 .../common/SplitFullScanEntityReader.java | 4 +-- .../service/generic/ListQueryResource.java | 10 +++ .../eagle/storage/operation/CompiledQuery.java | 16 +-- .../GenericCoprocessorAggregateQuery.java | 8 +++--- .../GenericInMemoryAggregateQuery.java | 8 +++--- 11 files changed, 57 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java -- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java index 8d6814b..e7ef3f8 100644 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java @@ -161,4 +161,12 @@ public class DateTimeUtil { Date d = sdf.parse(date); return d.getTime(); } + +public static long parseTimeStrToMilliseconds(String timeStr) throws ParseException { +try { +return Long.valueOf(timeStr); +} catch (Exception ex) { +return humanDateToSeconds(timeStr) * ONESECOND; +} +} } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java -- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java index 3f97e78..e0a7119 100755 --- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java +++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java @@ -80,8 +80,8 @@ public class GenericEntityScanStreamReader extends StreamReader { } // Process the time range if needed if(entityDef.isTimeSeries()){ - start = DateTimeUtil.humanDateToDate(condition.getStartTime()); - end = DateTimeUtil.humanDateToDate(condition.getEndTime()); + start = new Date(condition.getStartTime()); + end = new Date(condition.getEndTime()); }else{ start = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_START_HUMANTIME); end = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_END_HUMANTIME); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java -- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java index 1946d6c..bf72a36 100755 --- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java +++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java @@ -43,8 +43,8 @@ public class GenericEn
[6/6] incubator-eagle git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-eagle
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-eagle Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/30b127b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/30b127b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/30b127b8 Branch: refs/heads/master Commit: 30b127b8db310201acae1dd7801c14a8c8e62e78 Parents: 229d6b3 a267e73 Author: Hao Chen <h...@apache.org> Authored: Tue Nov 22 15:18:19 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Nov 22 15:18:19 2016 +0800 -- .../eagle/topology/TopologyCheckAppConfig.java | 9 ++-- .../hbase/HbaseTopologyEntityParser.java| 4 +- eagle.topology.TopologyCheckAppProvider.xml | 52 .../src/main/resources/application.conf | 9 ++-- 4 files changed, 33 insertions(+), 41 deletions(-) --
incubator-eagle git commit: [EAGLE-769] InjectMembers in ServerApplication when initialization
Repository: incubator-eagle Updated Branches: refs/heads/master 9795fbccb -> 8bb32c012 [EAGLE-769] InjectMembers in ServerApplication when initialization https://issues.apache.org/jira/browse/EAGLE-769 InjectMembers in ServerApplication when initialization Author: Hao Chen <h...@apache.org> Closes #654 from haoch/EAGLE-769. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8bb32c01 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8bb32c01 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8bb32c01 Branch: refs/heads/master Commit: 8bb32c012dd8b4c4f588824a54e4a2f0693762d9 Parents: 9795fbc Author: Hao Chen <h...@apache.org> Authored: Tue Nov 15 15:15:11 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Nov 15 15:15:11 2016 +0800 -- .../apache/eagle/server/ServerApplication.java | 211 ++- .../org/apache/eagle/server/ServerConfig.java | 6 - 2 files changed, 110 insertions(+), 107 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8bb32c01/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java -- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java index b9fa243..fb63530 100644 --- a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java +++ b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java @@ -1,102 +1,111 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.server; - -import com.google.inject.Injector; -import com.hubspot.dropwizard.guice.GuiceBundle; -import com.sun.jersey.api.core.PackagesResourceConfig; -import io.dropwizard.Application; -import io.dropwizard.assets.AssetsBundle; -import io.dropwizard.lifecycle.Managed; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; -import io.swagger.jaxrs.config.BeanConfig; -import io.swagger.jaxrs.listing.ApiListingResource; -import org.apache.eagle.alert.coordinator.CoordinatorListener; -import org.apache.eagle.alert.resource.SimpleCORSFiler; -import org.apache.eagle.app.service.ApplicationHealthCheckService; -import org.apache.eagle.common.Version; -import org.apache.eagle.log.base.taggedlog.EntityJsonModule; -import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; -import org.apache.eagle.metadata.service.ApplicationStatusUpdateService; -import org.apache.eagle.server.authentication.BasicAuthProviderBuilder; -import org.apache.eagle.server.task.ApplicationTask; -import org.apache.eagle.server.module.GuiceBundleLoader; - -import javax.servlet.DispatcherType; -import java.util.EnumSet; - -class ServerApplication extends Application { -private GuiceBundle guiceBundle; - -@Override -public void initialize(Bootstrap bootstrap) { -guiceBundle = GuiceBundleLoader.load(); -bootstrap.addBundle(guiceBundle); -bootstrap.addBundle(new AssetsBundle("/assets", "/", "index.html", "/")); -} - -@Override -public String getName() { -return ServerConfig.getServerName(); -} - -@Override -public void run(ServerConfig configuration, Environment environment) throws Exception { - environment.getApplicationContext().setContextPath(ServerConfig.getContextPath()); -environment.jersey().register(RESTExceptionMapper.class); -environment.jersey().setUrlPattern(ServerConfig.getApiBasePath()); - environment.getObjectMapper().setFilters(TaggedLogAPIEntity.getFilterProvider()); -environment.getObjectMapper().registerModule(new EntityJsonModule()); - -// Automatically scan all REST resources -new PackagesResourceConfig(ServerConfig.getResourcePackage()).getClasses().forEach(environment.jersey()::
incubator-eagle git commit: [EAGLE-766] Set default host & port of JPM_WEB_APP as null
Repository: incubator-eagle Updated Branches: refs/heads/master 41cb7e1a0 -> 44a4d660b [EAGLE-766] Set default host & port of JPM_WEB_APP as null https://issues.apache.org/jira/browse/EAGLE-766 Set default host & port of JPM_WEB_APP as N/A to make sure the web will read web service from configured instead of always using current. Author: Hao Chen <h...@apache.org> Closes #651 from haoch/EAGLE-766. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/44a4d660 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/44a4d660 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/44a4d660 Branch: refs/heads/master Commit: 44a4d660b3e32cb13ba7b4ffcfb21f2612abe449 Parents: 41cb7e1 Author: Hao Chen <h...@apache.org> Authored: Tue Nov 15 11:33:54 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Nov 15 11:33:54 2016 +0800 -- .../org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml | 6 ++ 1 file changed, 2 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/44a4d660/eagle-jpm/eagle-jpm-web/src/main/resources/META-INF/providers/org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml -- diff --git a/eagle-jpm/eagle-jpm-web/src/main/resources/META-INF/providers/org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml b/eagle-jpm/eagle-jpm-web/src/main/resources/META-INF/providers/org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml index 6620b21..222e306 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/resources/META-INF/providers/org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-web/src/main/resources/META-INF/providers/org.apache.eagle.app.jpm.JPMWebApplicationProvider.xml @@ -43,15 +43,13 @@ service.host -localhost Eagle Service Host -Eagle Service Host, default: localhost +Set additional eagle service host, default: using current host service.port -8080 Eagle Service Port -Eagle Service Port, default: 8080 +Set additional eagle service port, default: using current port \ No newline at end of file
incubator-eagle git commit: [EAGLE-643] Implement oozieAuditLogAppProvider
Repository: incubator-eagle Updated Branches: refs/heads/master 45af18e7f -> 8b504792b [EAGLE-643] Implement oozieAuditLogAppProvider EAGLE-643 Implement oozieAuditLogAppProvider - Add provider for oozie audit log https://issues.apache.org/jira/browse/EAGLE-643 Author: Lingang Deng <chitin1...@gmail.com> Author: chitin <chitin1...@gmail.com> Closes #606 from chitin/EAGLE-643. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8b504792 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8b504792 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8b504792 Branch: refs/heads/master Commit: 8b504792bbc551908be280cd50b357bebb8dad25 Parents: 45af18e Author: Lingang Deng <chitin1...@gmail.com> Authored: Mon Nov 14 10:57:25 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Nov 14 10:57:25 2016 +0800 -- .../oozie/parse/OozieAuditLogAppProvider.java | 31 +++ ...ity.oozie.parse.OozieAuditLogAppProvider.xml | 243 +++ ...org.apache.eagle.app.spi.ApplicationProvider | 16 ++ ...org.apache.eagle.app.spi.ApplicationProvider | 1 + 4 files changed, 291 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b504792/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogAppProvider.java -- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogAppProvider.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogAppProvider.java new file mode 100644 index 000..7430a9d --- /dev/null +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogAppProvider.java @@ -0,0 +1,31 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ +package org.apache.eagle.security.oozie.parse; + +import org.apache.eagle.app.spi.AbstractApplicationProvider; + +/** + * @Since 10/25/16. + */ +public class OozieAuditLogAppProvider extends AbstractApplicationProvider { + +public OozieAuditLogApplication getApplication() { +return new OozieAuditLogApplication(); +} +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b504792/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml -- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml new file mode 100644 index 000..6ebe4a2 --- /dev/null +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml @@ -0,0 +1,243 @@ + + + + +OOZIE_AUDIT_LOG_MONITOR +Oozie Audit Log Monitor +0.5.0-incubating + org.apache.eagle.security.oozie.parse.OozieAuditLogApplication + + + +workers +storm workers +1 +number of topology workers + + +topology.numOfJoinTasks +Topology Join Tasks +2 +number of join tasks + + +topology.numOfSpoutTasks +Topology Spout Tasks +2 +number of spout tasks + + +topology.numOfParserTasks +Topology Parser Tasks +2 +number of parser tasks + + +topology.numOfSensitivityJoinTa
[2/2] incubator-eagle git commit: [EAGLE-762] Add AlertEmailPublisherTest and improve alert template
[EAGLE-762] Add AlertEmailPublisherTest and improve alert template * Add AlertEmailPublisherTest * Improve alert template * Generate `alertId` in `AlertStreamEvent` instead of `AlertPublishEvent` to make sure alerts track-able among different publish plugin types. https://issues.apache.org/jira/browse/EAGLE-762 Author: Hao Chen <h...@apache.org> Closes #645 from haoch/EAGLE-762. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/38ec7fc2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/38ec7fc2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/38ec7fc2 Branch: refs/heads/master Commit: 38ec7fc23d06c77b6e083ad2dfb9c61e729901e0 Parents: 75ab772 Author: Hao Chen <h...@apache.org> Authored: Fri Nov 11 11:39:53 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Fri Nov 11 11:39:53 2016 +0800 -- .../alert/engine/model/AlertPublishEvent.java | 16 +- .../alert/engine/model/AlertStreamEvent.java| 24 +- .../service/MetadataServiceClientImpl.java | 596 +-- .../engine/model/AlertPublishEventTest.java | 1 + .../eagle-alert/alert-engine/pom.xml| 16 +- .../engine/publisher/PublishConstants.java | 12 +- .../publisher/email/AlertEmailGenerator.java| 67 ++- .../email/AlertEmailGeneratorBuilder.java | 10 + .../publisher/impl/AlertEmailPublisher.java | 14 +- .../publisher/impl/AlertPublisherImpl.java | 2 +- .../src/main/resources/ALERT_DEFAULT.vm | 94 +-- .../engine/metric/MemoryUsageGaugeSetTest.java | 2 +- .../publisher/AlertEmailPublisherTest.java | 76 +++ .../publisher/AlertKafkaPublisherTest.java | 123 +--- .../publisher/AlertPublisherTestHelper.java | 106 .../dedup/DefaultDedupWithoutStateTest.java | 240 +++- .../dedup/DefaultDeduplicatorTest.java | 71 +-- pom.xml | 4 +- 18 files changed, 780 insertions(+), 694 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/38ec7fc2/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java index 3453e55..6230731 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java @@ -18,9 +18,10 @@ package org.apache.eagle.alert.engine.model; +import com.google.common.base.Preconditions; + import java.util.List; import java.util.Map; -import java.util.UUID; public class AlertPublishEvent { private String alertId; @@ -29,7 +30,7 @@ public class AlertPublishEvent { private String policyId; private String policyValue; private long alertTimestamp; -private Map<String, String> alertData; +private Map<String, Object> alertData; public static final String SITE_ID_KEY = "siteId"; public static final String APP_IDS_KEY = "appIds"; @@ -84,17 +85,18 @@ public class AlertPublishEvent { this.policyId = policyId; } -public Map<String, String> getAlertData() { +public Map<String, Object> getAlertData() { return alertData; } -public void setAlertData(Map<String, String> alertData) { +public void setAlertData(Map<String, Object> alertData) { this.alertData = alertData; } public static AlertPublishEvent createAlertPublishEvent(AlertStreamEvent event) { +Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before being published: " + event.toString()); AlertPublishEvent alertEvent = new AlertPublishEvent(); -alertEvent.setAlertId(UUID.randomUUID().toString()); +alertEvent.setAlertId(event.getAlertId()); alertEvent.setPolicyId(event.getPolicyId()); alertEvent.setAlertTimestamp(event.getCreatedTime()); if (event.getExtraData() != null && !event.getExtraData().isEmpty()) { @@ -105,6 +107,4 @@ public class AlertPublishEvent { alertEvent.setAlertData(event.getDataMap()); return alertEvent; } - -} - +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/38ec7fc2
incubator-eagle git commit: [MINOR] Fix test fail on windows
Repository: incubator-eagle Updated Branches: refs/heads/master 3099a9e8c -> ad8cc9c99 [MINOR] Fix test fail on windows - Fix test fail on windows by change to END_LINE.length() Author: r7raul1984 <tangji...@yhd.com> Closes #641 from r7raul1984/EAGLE-733-MINOR. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ad8cc9c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ad8cc9c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ad8cc9c9 Branch: refs/heads/master Commit: ad8cc9c99bf33c75ebc1de60f87fc91128535039 Parents: 3099a9e Author: r7raul1984 <tangji...@yhd.com> Authored: Thu Nov 10 12:29:02 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Nov 10 12:29:02 2016 +0800 -- .../test/java/org/apache/eagle/alert/metric/MetricSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ad8cc9c9/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java index 6632f80..688e321 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java @@ -188,7 +188,7 @@ public class MetricSystemTest { sink.report(); sink.stop(); String result = bytes.toString(); -result = result.substring(result.indexOf(END_LINE) + 1);//remove first line +result = result.substring(result.indexOf(END_LINE) + END_LINE.length());//remove first line Assert.assertEquals("" + END_LINE + "" + "-- Gauges --" + END_LINE + "" + "name" + END_LINE + "" +
incubator-eagle git commit: [EAGLE-734] Integrate maven-buildNumber-plugin and Version.java
Repository: incubator-eagle Updated Branches: refs/heads/master 32582d230 -> 3099a9e8c [EAGLE-734] Integrate maven-buildNumber-plugin and Version.java * Integrate maven-buildNumber-plugin and Version.java * Format eagle version desc * Add `/server/version` and `/server/config` API Author: Hao Chen <h...@apache.org> Closes #639 from haoch/buildNumberVersion. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3099a9e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3099a9e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3099a9e8 Branch: refs/heads/master Commit: 3099a9e8ccabd3c371afe8f61fd8a1a151083232 Parents: 32582d2 Author: Hao Chen <h...@apache.org> Authored: Thu Nov 10 12:26:22 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Nov 10 12:26:22 2016 +0800 -- eagle-core/eagle-common/pom.xml | 43 ++ .../org/apache/eagle/common/Version.java| 59 .../org/apache/eagle/common/TestVersion.java| 27 + .../apache/eagle/server/ServerApplication.java | 6 +- .../org/apache/eagle/server/ServerConfig.java | 22 ++-- .../org/apache/eagle/server/ServerMain.java | 9 ++- .../server/managedtask/ApplicationTask.java | 43 -- .../eagle/server/resource/ServerResource.java | 48 .../eagle/server/task/ApplicationTask.java | 42 ++ pom.xml | 16 +- 10 files changed, 262 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3099a9e8/eagle-core/eagle-common/pom.xml -- diff --git a/eagle-core/eagle-common/pom.xml b/eagle-core/eagle-common/pom.xml index 26b8f8f..933d4e2 100644 --- a/eagle-core/eagle-common/pom.xml +++ b/eagle-core/eagle-common/pom.xml @@ -109,6 +109,49 @@ true + +org.codehaus.mojo +buildnumber-maven-plugin + + +1 +validate + +create +create-metadata + + + + +{0,number} + +buildNumber + +false + true +true +true + +${buildNumber} +${user.name} + +false +unknown + + + +org.codehaus.mojo +templating-maven-plugin + + +filtering-java-templates +generate-sources + +filter-sources + + + + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3099a9e8/eagle-core/eagle-common/src/main/java-templates/org/apache/eagle/common/Version.java -- diff --git a/eagle-core/eagle-common/src/main/java-templates/org/apache/eagle/common/Version.java b/eagle-core/eagle-common/src/main/java-templates/org/apache/eagle/common/Version.java new file mode 100644 index 000..0b84211 --- /dev/null +++ b/eagle-core/eagle-common/src/main/java-templates/org/apache/eagle/common/Version.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.common; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; +import java.util.Prop
[2/2] incubator-eagle git commit: [EAGLE-738] Fix eagle-metadata checkstyle warnings
[EAGLE-738] Fix eagle-metadata checkstyle warnings [EAGLE-738]Fix eagle-metadata checkstyle warnings https://issues.apache.org/jira/browse/EAGLE-738 Author: koone <luokun1...@126.com> Closes #636 from koone/EAGLE-738. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/32582d23 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/32582d23 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/32582d23 Branch: refs/heads/master Commit: 32582d2300306d6321fcd026b62b17ff1707d95c Parents: a88e1ce Author: koone <luokun1...@126.com> Authored: Wed Nov 9 22:01:07 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 22:01:07 2016 +0800 -- .../exceptions/EntityNotFoundException.java | 2 +- .../metadata/model/ApplicationDependency.java | 2 +- .../eagle/metadata/model/ApplicationDesc.java | 2 +- .../eagle/metadata/model/ApplicationDocs.java | 4 +- .../eagle/metadata/model/ApplicationEntity.java | 12 +- .../metadata/model/ApplicationRawEntity.java| 4 +- .../apache/eagle/metadata/model/SiteEntity.java | 2 +- .../eagle/metadata/model/StreamSinkConfig.java | 2 + .../metadata/persistence/MetadataStore.java | 2 +- .../persistence/MetadataStoreModuleFactory.java | 14 +-- .../metadata/persistence/PersistenceEntity.java | 14 +-- .../persistence/PersistenceService.java | 2 + .../eagle/metadata/resource/RESTResponse.java | 117 ++- .../eagle/metadata/resource/SiteResource.java | 30 ++--- .../eagle/metadata/resource/UUIDRequest.java| 2 +- .../service/ApplicationDescService.java | 1 + .../service/ApplicationEntityService.java | 8 +- .../service/ApplicationStatusUpdateService.java | 3 +- .../metadata/service/SiteEntityService.java | 5 +- .../ApplicationEntityServiceMemoryImpl.java | 2 +- .../service/memory/MemoryMetadataStore.java | 2 +- .../SiteEntityEntityServiceMemoryImpl.java | 26 ++--- .../metadata/utils/ConfigTemplateHelper.java| 26 ++--- .../metadata/utils/InnerDomAsTextHandler.java | 8 +- .../eagle/metadata/utils/UUIDGenerator.java | 2 +- .../metadata/utils/StreamIdConversionsTest.java | 4 +- .../store/jdbc/JDBCDataSourceConfig.java| 36 +++--- .../JDBCMetadataMetadataStoreServiceImpl.java | 8 +- .../store/jdbc/JDBCMetadataQueryService.java| 46 .../metadata/store/jdbc/JDBCMetadataStore.java | 3 +- .../jdbc/provider/JDBCDataSourceProvider.java | 8 +- .../JDBCMetadataStoreConfigProvider.java| 6 +- .../ApplicationEntityServiceJDBCImpl.java | 16 ++- .../jdbc/service/SiteEntityServiceJDBCImpl.java | 6 +- .../orm/ApplicationEntityToRelation.java| 3 +- .../orm/RelationToApplicationEntity.java| 2 +- .../jdbc/service/orm/SiteEntityToRelation.java | 3 +- .../ApplicationEntityServiceJDBCImplTest.java | 1 + .../store/jdbc/JDBCDataSourceProviderTest.java | 29 ++--- .../jdbc/JDBCMetadataQueryServiceTest.java | 6 +- .../store/jdbc/JDBCMetadataTestBase.java| 12 +- eagle-core/eagle-metadata/pom.xml | 12 ++ 42 files changed, 234 insertions(+), 261 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/32582d23/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/exceptions/EntityNotFoundException.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/exceptions/EntityNotFoundException.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/exceptions/EntityNotFoundException.java index 990d77c..f23785a 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/exceptions/EntityNotFoundException.java +++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/exceptions/EntityNotFoundException.java @@ -16,7 +16,7 @@ */ package org.apache.eagle.metadata.exceptions; -public class EntityNotFoundException extends Exception{ +public class EntityNotFoundException extends Exception { public EntityNotFoundException(Throwable cause) { super(cause); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/32582d23/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDependency.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDependency.java b/eagle-core/eagle-metadata/eagle-meta
[1/2] incubator-eagle git commit: [EAGLE-738] Fix eagle-metadata checkstyle warnings
Repository: incubator-eagle Updated Branches: refs/heads/master a88e1cef8 -> 32582d230 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/32582d23/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataTestBase.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataTestBase.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataTestBase.java index 3315fc3..adfe8f2 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataTestBase.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataTestBase.java @@ -19,13 +19,9 @@ package org.apache.eagle.metadata.store.jdbc; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; -import com.typesafe.config.ConfigFactory; import org.apache.commons.dbcp.BasicDataSource; import org.apache.eagle.app.module.ApplicationGuiceModule; -import org.apache.eagle.app.service.ApplicationProviderService; -import org.apache.eagle.app.service.impl.ApplicationProviderServiceImpl; import org.apache.eagle.common.module.CommonGuiceModule; -import org.apache.eagle.metadata.store.jdbc.JDBCMetadataStore; import org.junit.After; import org.junit.Before; @@ -39,19 +35,19 @@ public class JDBCMetadataTestBase { private DataSource dataSource; @Before -public void setUp(){ -injector = Guice.createInjector(new JDBCMetadataStore(),new ApplicationGuiceModule(),new CommonGuiceModule()); +public void setUp() { +injector = Guice.createInjector(new JDBCMetadataStore(), new ApplicationGuiceModule(), new CommonGuiceModule()); injector.injectMembers(this); } @After public void after() throws SQLException { -if(dataSource!=null){ +if (dataSource != null) { ((BasicDataSource) dataSource).close(); } } -public Injector injector(){ +public Injector injector() { return injector; } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/32582d23/eagle-core/eagle-metadata/pom.xml -- diff --git a/eagle-core/eagle-metadata/pom.xml b/eagle-core/eagle-metadata/pom.xml index 676327e..57dde36 100644 --- a/eagle-core/eagle-metadata/pom.xml +++ b/eagle-core/eagle-metadata/pom.xml @@ -33,4 +33,16 @@ eagle-metadata-mongo eagle-metadata-base + + + +org.apache.maven.plugins +maven-checkstyle-plugin + +true +true + + + + \ No newline at end of file
[2/2] incubator-eagle git commit: [EAGLE-737] Fix eagle-embed checkstyle warnings
[EAGLE-737] Fix eagle-embed checkstyle warnings [EAGLE-737]Fix eagle-embed checkstyle warnings https://issues.apache.org/jira/browse/EAGLE-737 Author: koone <luokun1...@126.com> Closes #634 from koone/EAGLE-737. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a88e1cef Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a88e1cef Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a88e1cef Branch: refs/heads/master Commit: a88e1cef80b3f9c6e896d5948154cd5b5b37d643 Parents: 2b61cef Author: koone <luokun1...@126.com> Authored: Wed Nov 9 21:58:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 21:58:55 2016 +0800 -- .../eagle/service/hbase/EmbeddedHbase.java | 30 +- .../org/apache/eagle/service/hbase/Tables.java |2 +- .../src/main/resources/hbase-default.xml| 1809 +- .../eagle/service/hbase/TestHBaseBase.java | 10 +- .../embedded/tomcat/TestEmbeddedServer.java | 12 +- eagle-core/eagle-embed/pom.xml | 12 + 6 files changed, 947 insertions(+), 928 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a88e1cef/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java -- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java index be614d3..84661e9 100644 --- a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java +++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java @@ -27,21 +27,21 @@ public class EmbeddedHbase { private HBaseTestingUtility util; private MiniHBaseCluster hbaseCluster; private static EmbeddedHbase hbase; -private int port; +private int port; private String znode; private static int DEFAULT_PORT = 2181; private static String DEFAULT_ZNODE = "/hbase-unsecure"; private static final Logger LOG = LoggerFactory.getLogger(EmbeddedHbase.class); - + private EmbeddedHbase(int port, String znode) { this.port = port; this.znode = znode; } - + private EmbeddedHbase(int port) { this(port, DEFAULT_ZNODE); } - + public static EmbeddedHbase getInstance(Configuration conf) { if (hbase == null) { synchronized (EmbeddedHbase.class) { @@ -55,7 +55,7 @@ public class EmbeddedHbase { } public static EmbeddedHbase getInstance() { -return getInstance(null); +return getInstance(null); } private EmbeddedHbase() { @@ -70,7 +70,7 @@ public class EmbeddedHbase { try { util = new HBaseTestingUtility(); Configuration conf = util.getConfiguration(); -if(confMap != null) { +if (confMap != null) { conf.addResource(confMap); } conf.setInt("test.hbase.zookeeper.property.clientPort", port); @@ -81,29 +81,29 @@ public class EmbeddedHbase { // start mini hbase cluster hbaseCluster = util.startMiniCluster(); Configuration config = hbaseCluster.getConf(); - + config.set("zookeeper.session.timeout", "12"); config.set("hbase.zookeeper.property.tickTime", "6000"); config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1"); config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "6"); - + Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { shutdown(); } -}); +}); } catch (Throwable t) { -LOG.error("Got an exception: ",t); +LOG.error("Got an exception: ", t); } } -public void shutdown() { +public void shutdown() { try { util.shutdownMiniCluster(); } catch (Throwable t) { -LOG.info("Got an exception, " + t , t.getCause()); +LOG.info("Got an exception, " + t, t.getCause()); try { util.shutdownMiniCluster(); } catch (Throwable t1) { @@ -111,15 +111,15 @@ public class Embedded
[1/2] incubator-eagle git commit: [EAGLE-737] Fix eagle-embed checkstyle warnings
Repository: incubator-eagle Updated Branches: refs/heads/master 2b61cef58 -> a88e1cef8 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a88e1cef/eagle-core/eagle-embed/pom.xml -- diff --git a/eagle-core/eagle-embed/pom.xml b/eagle-core/eagle-embed/pom.xml index b05b798..10f2848 100644 --- a/eagle-core/eagle-embed/pom.xml +++ b/eagle-core/eagle-embed/pom.xml @@ -33,4 +33,16 @@ eagle-embed-hbase eagle-embed-server + + + +org.apache.maven.plugins +maven-checkstyle-plugin + +true +true + + + +
[2/2] incubator-eagle git commit: [EAGLE-749] Add unit test for model.
[EAGLE-749] Add unit test for model. - Add unit test for model which under alert-common moudle. - Fix equals() hashcode() inconsistent for PartitionedEvent,StreamEvent,Publishment. - Fix bug for StreamColumn BooleanType. https://issues.apache.org/jira/browse/EAGLE-749 Author: r7raul1984 <tangji...@yhd.com> Closes #632 from r7raul1984/EAGLE-749. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2b61cef5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2b61cef5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2b61cef5 Branch: refs/heads/master Commit: 2b61cef585b02fb6f40d5c3de7f7c5c060926ecd Parents: 1da8dc4 Author: r7raul1984 <tangji...@yhd.com> Authored: Wed Nov 9 21:55:51 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 21:55:51 2016 +0800 -- .../alert/engine/coordinator/Publishment.java | 21 +- .../alert/engine/coordinator/StreamColumn.java | 2 +- .../engine/coordinator/StreamSortSpec.java | 17 +- .../alert/engine/model/PartitionedEvent.java| 16 +- .../eagle/alert/engine/model/StreamEvent.java | 11 +- .../eagle/alert/config/TestConfigBus.java | 5 +- .../model/Kafka2TupleMetadataTest.java | 49 + .../model/PolicyWorkerQueueTest.java| 63 ++ .../model/StreamRepartitionStrategyTest.java| 74 +++ .../model/StreamRouterSpecTest.java | 59 ++ .../model/Tuple2StreamMetadataTest.java | 50 + .../alert/coordination/model/WorkSlotTest.java | 43 + .../model/internal/MonitoredStreamTest.java | 69 +++ .../model/internal/PolicyAssignmentTest.java| 37 .../model/internal/StreamGroupTest.java | 67 +++ .../model/internal/StreamWorkSlotQueueTest.java | 61 ++ .../model/internal/TopologyTest.java| 47 + .../OverrideDeduplicatorSpecTest.java | 61 ++ .../coordinator/PolicyDefinitionTest.java | 140 ++ .../engine/coordinator/PublishmentTest.java | 100 ++ .../engine/coordinator/PublishmentTypeTest.java | 47 + .../engine/coordinator/StreamColumnTest.java| 153 +++ .../coordinator/StreamDefinitionTest.java | 52 + .../engine/coordinator/StreamPartitionTest.java | 43 + .../engine/coordinator/StreamSortSpecTest.java | 45 + .../coordinator/StreamingClusterTest.java | 47 + .../engine/model/AlertPublishEventTest.java | 110 +++ .../engine/model/AlertStreamEventTest.java | 63 ++ .../engine/model/PartitionedEventTest.java | 54 ++ .../engine/model/StreamEventBuilderTest.java| 166 .../alert/engine/model/StreamEventTest.java | 191 +++ .../eagle/alert/model/StreamEventTest.java | 68 --- .../eagle/alert/model/TestPolicyDefinition.java | 45 - 33 files changed, 1930 insertions(+), 146 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2b61cef5/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java index dbb1844..d1cc33a 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java @@ -135,28 +135,29 @@ public class Publishment { if (obj instanceof Publishment) { Publishment p = (Publishment) obj; return (Objects.equals(name, p.getName()) && Objects.equals(type, p.getType()) -&& Objects.equals(dedupIntervalMin, p.getDedupIntervalMin()) -&& Objects.equals(dedupFields, p.getDedupFields()) -&& Objects.equals(dedupStateField, p.getDedupStateField()) -&& Objects.equals(overrideDeduplicator, p.getOverrideDeduplicator()) -&& Objects.equals(policyIds, p.getPolicyIds()) -&& Objects.equals(streamIds, p.getStreamIds()) -&& properties.equals(p.getProperties())); +&& Objects.equals(dedupIntervalMin, p.getDedupIntervalMin()) +&& Objects.equals(dedupFields
[1/2] incubator-eagle git commit: [EAGLE-749] Add unit test for model.
Repository: incubator-eagle Updated Branches: refs/heads/master 1da8dc4f1 -> 2b61cef58 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2b61cef5/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamPartitionTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamPartitionTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamPartitionTest.java new file mode 100644 index 000..f7615df --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamPartitionTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.alert.engine.coordinator; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class StreamPartitionTest { +@Test +public void testStreamPartition() { +StreamSortSpec streamSortSpec = new StreamSortSpec(); +streamSortSpec.setWindowPeriod("PT10S"); +StreamPartition streamPartition = new StreamPartition(); + Assert.assertEquals("StreamPartition[streamId=null,type=null,columns=[],sortSpec=[null]]", streamPartition.toString()); +List columns = new ArrayList<>(); +columns.add("jobId"); +streamPartition.setColumns(columns); +streamPartition.setSortSpec(streamSortSpec); +streamPartition.setStreamId("test"); +streamPartition.setType(StreamPartition.Type.GROUPBY); + Assert.assertEquals("StreamPartition[streamId=test,type=GROUPBY,columns=[jobId],sortSpec=[StreamSortSpec[windowPeriod=PT10S,windowMargin=3]]]", streamPartition.toString()); +Assert.assertTrue(streamPartition.equals(new StreamPartition(streamPartition))); +Assert.assertTrue(streamPartition.hashCode() == new StreamPartition(streamPartition).hashCode()); +} +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2b61cef5/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpecTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpecTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpecTest.java new file mode 100644 index 000..d8123f1 --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpecTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.alert.engine.coordinator; + +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +public class StreamSortSpecTest { +@Test +public void testStreamSortSpec() { +StreamSortSpec streamSortSpec = new StreamSortSpec(); +Assert.assertEquals(3, streamSortSpec.getWindowMargin()); +Assert.assertEquals("", streamSortSpec.getWindowPeriod()); +Assert.assertEquals(0,
incubator-eagle git commit: [EAGLE-759] Policy detail support alert list
Repository: incubator-eagle Updated Branches: refs/heads/master db28ae2b2 -> 1da8dc4f1 [EAGLE-759] Policy detail support alert list * Policy detail support alert list * update site application logic Author: zombieJ <smith3...@gmail.com> Closes #635 from zombieJ/EAGLE-759. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/1da8dc4f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/1da8dc4f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/1da8dc4f Branch: refs/heads/master Commit: 1da8dc4f1fbb46c5ce76cd3b2af7f680f6f8cc29 Parents: db28ae2 Author: zombieJ <smith3...@gmail.com> Authored: Wed Nov 9 21:20:02 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 21:20:02 2016 +0800 -- .../webapp/app/apps/jpm/ctrl/statisticCtrl.js | 2 +- .../src/main/webapp/app/apps/jpm/index.js | 6 +- .../webapp/app/dev/partials/alert/list.html | 24 +++- .../app/dev/partials/alert/policyDetail.html| 59 +--- .../webapp/app/dev/public/js/ctrls/alertCtrl.js | 18 +- .../app/dev/public/js/ctrls/integrationCtrl.js | 25 ++--- 6 files changed, 110 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1da8dc4f/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/statisticCtrl.js -- diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/statisticCtrl.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/statisticCtrl.js index 6dff7a1..e74a22c 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/statisticCtrl.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/statisticCtrl.js @@ -354,7 +354,7 @@ * @param {[]} res.data.jobTypes * @param {[]} res.data.jobCounts */ - JPM.jobDistribution($scope.site, $scope.type, DURATION_BUCKETS.join(","), startTime, endTime).then(function (res) { + JPM.jobDistribution($scope.site, DURATION_BUCKETS.join(","), startTime, endTime).then(function (res) { var data = res.data; var jobTypes = {}; $.each(data.jobTypes, function (i, type) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1da8dc4f/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js -- diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js index fafe699..7348853 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js @@ -75,7 +75,7 @@ JPM.QUERY_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}=${metric}=${limit}=${startTime}=${endTime}=${intervalMin}=true'; JPM.QUERY_MR_JOBS = '${baseURL}/rest/mrJobs/search'; JPM.QUERY_JOB_LIST = '${baseURL}/rest/mrJobs?query=%s[${condition}]{${fields}}=${limit}=${startTime}=${endTime}'; - JPM.QUERY_JOB_STATISTIC = '${baseURL}/rest/mrJobs/jobCountsByDuration?site=${site}=${times}=${startTime}=${endTime}=${jobType}'; + JPM.QUERY_JOB_STATISTIC = '${baseURL}/rest/mrJobs/jobCountsByDuration?site=${site}=${times}=${startTime}=${endTime}'; JPM.QUERY_TASK_STATISTIC = '${baseURL}/rest/mrTasks/taskCountsByDuration?jobId=${jobId}=${site}=${times}=${top}'; JPM.QUERY_MR_JOB_COUNT = '${baseURL}/rest/mrJobs/runningJobCounts'; @@ -354,15 +354,13 @@ /** * Fetch job duration distribution * @param {string} site -* @param {string} jobType * @param {string} times * @param {{}} startTime * @param {{}} endTime */ - JPM.jobDistribution = function (site, jobType, times, startTime, endTime) { + JPM.jobDistribution = function (site, times, startTime, endTime) { var url = common.template(getQuery("JOB_STATISTIC"), { site: site, - jobType: jobType, times: times, startTime: Time.format(startTi
incubator-eagle git commit: [EAGLE-739] Fix eagle-metric checkstyle warnings
Repository: incubator-eagle Updated Branches: refs/heads/master 2f4df34cf -> 4df63def8 [EAGLE-739] Fix eagle-metric checkstyle warnings [EAGLE-739]Fix eagle-metric checkstyle warnings https://issues.apache.org/jira/browse/EAGLE-739 Author: koone <luokun1...@126.com> Closes #637 from koone/EAGLE-739. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4df63def Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4df63def Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4df63def Branch: refs/heads/master Commit: 4df63def83f1e33fd0899693d5ac751b24bf39a2 Parents: 2f4df34 Author: koone <luokun1...@126.com> Authored: Wed Nov 9 18:58:18 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 18:58:18 2016 +0800 -- eagle-core/eagle-metric/pom.xml | 13 + .../eagle/metric/reportor/EagleCounterMetric.java| 5 ++--- .../eagle/metric/reportor/EagleGaugeMetric.java | 3 --- .../eagle/metric/reportor/EagleMetricListener.java | 3 ++- .../reportor/EagleServiceReporterMetricListener.java | 11 +-- .../apache/eagle/metric/reportor/IEagleMetric.java | 5 +++-- .../eagle/metric/reportor/MetricEntityAdaptor.java | 11 +-- .../eagle/metric/reportor/MetricKeyCodeDecoder.java | 15 --- .../eagle-metric/src/test/resources/log4j.properties | 3 --- 9 files changed, 38 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4df63def/eagle-core/eagle-metric/pom.xml -- diff --git a/eagle-core/eagle-metric/pom.xml b/eagle-core/eagle-metric/pom.xml index e4dee1a..225ecee 100644 --- a/eagle-core/eagle-metric/pom.xml +++ b/eagle-core/eagle-metric/pom.xml @@ -52,4 +52,17 @@ test + + + + +org.apache.maven.plugins +maven-checkstyle-plugin + +true +true + + + + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4df63def/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleCounterMetric.java -- diff --git a/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleCounterMetric.java b/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleCounterMetric.java index 0a7f70e..5251b46 100644 --- a/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleCounterMetric.java +++ b/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleCounterMetric.java @@ -44,7 +44,7 @@ public class EagleCounterMetric extends EagleMetric { for (EagleMetricListener listener : metricListeners) { EagleCounterMetric newEagleMetric = new EagleCounterMetric(this); newEagleMetric.name = MetricKeyCodeDecoder.addTimestampToMetricKey(trim(latestUserTimeClock), newEagleMetric.name); - listener.onMetricFlushed(Arrays.asList((EagleMetric)newEagleMetric)); +listener.onMetricFlushed(Arrays.asList((EagleMetric) newEagleMetric)); } } @@ -62,8 +62,7 @@ public class EagleCounterMetric extends EagleMetric { LOG.warn("Something must be wrong, event should come in order of userTimeClock"); } value.addAndGet(d); -} -else { +} else { flush(latestUserTimeClock); value.getAndSet(1); latestUserTimeClock = currentUserTimeClock; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4df63def/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleGaugeMetric.java -- diff --git a/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleGaugeMetric.java b/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleGaugeMetric.java index e6fc098..3d75d32 100644 --- a/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleGaugeMetric.java +++ b/eagle-core/eagle-metric/src/main/java/org/apache/eagle/metric/reportor/EagleGaugeMetric.java @@ -22,9 +22,6 @@ package org.apache.eagle.metric.reportor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.Map; - public class EagleGaugeMetric extends EagleMetric { private static final Logger LOG = LoggerFactory.getLogger(EagleGaugeMetric.class); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4df63
incubator-eagle git commit: [EAGLE-698] Collectd python plugin for gathering hadoop jmx information.
Repository: incubator-eagle Updated Branches: refs/heads/master 327351b92 -> 2f4df34cf [EAGLE-698] Collectd python plugin for gathering hadoop jmx information. Be sure to do all of the following to help us incorporate your contribution quickly and easily: - [ ] Make sure the PR title is formatted like: `[EAGLE-] Description of pull request` - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable Travis-CI on your fork and ensure the whole test matrix passes). - [ ] Replace `` in the title with the actual Jira issue number, if there is one. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt). --- â¦rmation. - hadooproleconfig.json is a configuration file for hadoop roles to be collect. - Manual_of_collectd_hadoop_plugin.md is a how-to document - https://issues.apache.org/jira/browse/EAGLE-698 Author: joe-hj <joe.h...@gmail.com> Closes #584 from joe-hj/EAGLE-698. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2f4df34c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2f4df34c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2f4df34c Branch: refs/heads/master Commit: 2f4df34cfbcce2ab0cd72f52c45a7873ec689722 Parents: 327351b Author: joe-hj <joe.h...@gmail.com> Authored: Wed Nov 9 17:01:22 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 17:01:22 2016 +0800 -- eagle-external/hadoop_jmx_collectd/README.md| 63 +++ eagle-external/hadoop_jmx_collectd/hadoop.py| 167 +++ .../hadoop_jmx_collectd/hadooproleconfig.json | 57 +++ 3 files changed, 287 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2f4df34c/eagle-external/hadoop_jmx_collectd/README.md -- diff --git a/eagle-external/hadoop_jmx_collectd/README.md b/eagle-external/hadoop_jmx_collectd/README.md new file mode 100755 index 000..beab2c9 --- /dev/null +++ b/eagle-external/hadoop_jmx_collectd/README.md @@ -0,0 +1,63 @@ +#**A manual for hadoop plugin of collectd** + +Collectd website referto [collectd.org](collectd.org) + +###Description +The plugin collect information from http://hostname:port/jmx , according to the hadoop role configuration, it support roles as below: + + HDFS NameNode + HDFS DataNode + HDFS JournalNode + HBase Master + Hbase RegionServer + Yarn NodeManager + Yarn ResourceManager + +###Install +>1) +Deploy hadoop.py in your collectd plugins path , in my environment, it is like : /opt/collectd/lib/collectd/plugins/hadoop.py(It assume that you install collectd in the directory of /opt/collectd), maybe you should create the plugins directory ahead. + +>2) +Configure the python plugin in file collectd.conf , and you should trun on the python switch first. + +>A snippet of collectd.conf for showing hadoop python plugin configuration: +> +LoadPlugin python + + ModulePath "/opt/collectd/lib/collectd/plugins/" +LogTraces true +Import "hadoop" + +HDFSDatanodeHost "YourHostName" +Port "50075" +Verbose true +Instance "192.168.xxx.xxx" +JsonPath "/xxx/xxx/hadooproleconfig.json" + + +YARNResourceManager "YourHostName" +Port "8088" +Verbose true +Instance "192.168.xxx.xxx" + + + +>**Notification**: +>Instance , port and host(role) fileds must be set. + +>3)Two way to cite hadooproleconfig.json , either is ok. +>>a)Place hadooproleconfig.json path in onepair in collectd.conf. +>>b)Place hadooproleconfig.json file in you BaseDir which defined in collectd.conf. + +>4) +If you update your collectd.conf or hadooproleconfig.json, you should restart your collectd application. + +###Dependency + collectd: I test in version 5.6.0. + Hadoop: I test in version 2.6.0-cdh5.4.3. + Hbase: I test in version 1.0.0-chd5.4.3. + +###Testing +>You can use hadoop.py as a single independent python file for debugging jmx, and you can also use it as plugin for collectd, a variable MyDebug is used as a swtich when used in the two different environments. + + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2f
incubator-eagle git commit: [EAGLE-747] Add unit test for eagle-storage-base module
Repository: incubator-eagle Updated Branches: refs/heads/master 137b9e038 -> 53a74633e [EAGLE-747] Add unit test for eagle-storage-base module EAGLE-747 Add unit test for eagle-storage-base module - Add unit test. https://issues.apache.org/jira/browse/EAGLE-747 Author: chitin <chitin1...@gmail.com> Closes #620 from chitin/EAGLE-747. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/53a74633 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/53a74633 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/53a74633 Branch: refs/heads/master Commit: 53a74633e90c9638554a96b7d21ef1a0bbbd4755 Parents: 137b9e0 Author: chitin <chitin1...@gmail.com> Authored: Wed Nov 9 15:53:06 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 15:53:06 2016 +0800 -- .../storage/operation/TestCompiledQuery.java| 81 .../storage/operation/TestCreateStatement.java | 53 + .../storage/operation/TestDeleteStatement.java | 68 .../storage/operation/TestQueryStatement.java | 41 ++ .../operation/TestRowkeyQueryStatement.java | 52 + .../storage/operation/TestUpdateStatement.java | 47 6 files changed, 342 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/53a74633/eagle-core/eagle-query/eagle-storage-base/src/test/java/org/apache/eagle/storage/operation/TestCompiledQuery.java -- diff --git a/eagle-core/eagle-query/eagle-storage-base/src/test/java/org/apache/eagle/storage/operation/TestCompiledQuery.java b/eagle-core/eagle-query/eagle-storage-base/src/test/java/org/apache/eagle/storage/operation/TestCompiledQuery.java new file mode 100644 index 000..a59a5d8 --- /dev/null +++ b/eagle-core/eagle-query/eagle-storage-base/src/test/java/org/apache/eagle/storage/operation/TestCompiledQuery.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.storage.operation; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.common.config.EagleConfigFactory; +import org.apache.eagle.log.entity.meta.EntityDefinition; +import org.apache.eagle.log.entity.meta.EntityDefinitionManager; +import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity; +import org.apache.eagle.storage.exception.QueryCompileException; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.GregorianCalendar; + +/** + * @Since 11/7/16. + */ +public class TestCompiledQuery { + +static EntityDefinition entityDefinition = null; + +final String queryStr = "TestTimeSeriesAPIEntity[@cluster=\"c4ut\"]{*}"; +final int pageSize = 1000; +final long baseTimestamp = createTimeStamp(2017, 0, 11, 0, 0, 0); +final String startTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp); +final String endTime = DateTimeUtil.millisecondsToHumanDateWithMilliseconds(baseTimestamp + 2000); + +@BeforeClass +public static void setUpOnce() throws Exception { +entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class); +entityDefinition.setTags(new String[] {"cluster", "datacenter", "random"}); +} + +static long createTimeStamp(int year, int month, int date, int hourOfDay, int minute, int second) { +GregorianCalendar gc = new GregorianCalendar(); +gc.clear(); +gc.set(year, month, date, hourOfDay, minute, second); +gc.setTimeZone(EagleConfigFactory.load().getTimeZone()); +return gc.getTime().getTime(); +} + +@Test +public void testCreatedCompiledQuery() throws QueryCompileException, IOException { + + +RawQuery rawQuery = new RawQuery(); +rawQuery.setQuery
incubator-eagle git commit: [EAGLE-733] Add unit test for MetricSystem.
Repository: incubator-eagle Updated Branches: refs/heads/master 81114c3b5 -> 137b9e038 [EAGLE-733] Add unit test for MetricSystem. - Add unit test for MetricSystem. - Add unit test for util. - Fix some oozie module check-style problem. https://issues.apache.org/jira/browse/EAGLE-733 Author: r7raul1984 <tangji...@yhd.com> Closes #619 from r7raul1984/EAGLE-733. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/137b9e03 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/137b9e03 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/137b9e03 Branch: refs/heads/master Commit: 137b9e038e8fe056e003f2b39e8385cdd63db160 Parents: 81114c3 Author: r7raul1984 <tangji...@yhd.com> Authored: Wed Nov 9 15:16:32 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 9 15:16:32 2016 +0800 -- .../eagle-alert/alert-common/pom.xml| 7 +- .../apache/eagle/alert/utils/KafkaEmbedded.java | 6 + .../eagle/alert/metric/MetricSystemTest.java| 379 ++- .../eagle/alert/util/ConfigUtilsTest.java | 76 .../apache/eagle/alert/util/JsonUtilsTest.java | 47 +++ .../alert/util/StreamIdConversionTest.java | 46 +++ .../eagle/alert/util/TimePeriodUtilsTest.java | 100 + .../PartitionedEventSerializerTest.java | 7 - .../alert/engine/utils/TimePeriodUtilsTest.java | 91 - .../res/OozieMetadataBrowseWebResource.java | 4 +- 10 files changed, 642 insertions(+), 121 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/137b9e03/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml index 7edd01b..0fba528 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml @@ -130,7 +130,12 @@ com.sun.jersey jersey-client - + +org.mockito +mockito-all +${mockito.version} +test + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/137b9e03/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/KafkaEmbedded.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/KafkaEmbedded.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/KafkaEmbedded.java index 7ec5872..46ca0a7 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/KafkaEmbedded.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/KafkaEmbedded.java @@ -20,6 +20,7 @@ import kafka.server.KafkaConfig; import kafka.server.KafkaServerStartable; import org.apache.commons.io.FileUtils; import org.apache.curator.test.InstanceSpec; + import java.io.File; import java.util.Properties; @@ -71,6 +72,10 @@ public class KafkaEmbedded { return new KafkaConfig(p); } +public String getZkConnectionString() { +return zk.getConnectionString(); +} + public String getBrokerConnectionString() { return "localhost:" + port; } @@ -89,4 +94,5 @@ public class KafkaEmbedded { } zk.shutdown(); } + } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/137b9e03/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java index 48bce2e..6632f80 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java @@ -16,42 +16,382 @@ */ package org.apache.eagle.alert.metric; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.JvmAttributeGaugeSet; -import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.*; +import com.codahale.metrics.jvm.Fi
incubator-eagle git commit: [MINOR] Resolve apache license header missing
Repository: incubator-eagle Updated Branches: refs/heads/master d4be077b8 -> 9f64be00e [MINOR] Resolve apache license header missing Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9f64be00 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9f64be00 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9f64be00 Branch: refs/heads/master Commit: 9f64be00e9d0e9156e0a187650a3ef9b4185d5b6 Parents: d4be077 Author: Hao Chen <h...@apache.org> Authored: Mon Nov 7 16:32:16 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Nov 7 16:32:16 2016 +0800 -- .../eagle/storage/hbase/spi/TestHBaseStorage.java | 16 .../TestHBaseStorageAggregateWithCoprocessor.java | 16 2 files changed, 32 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f64be00/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java -- diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java index 2a497d1..de24b5d 100755 --- a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java +++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.eagle.storage.hbase.spi; import org.apache.eagle.common.DateTimeUtil; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f64be00/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java -- diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java index 1f27a72..8620a5b 100755 --- a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java +++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.eagle.storage.hbase.spi; import org.apache.eagle.common.DateTimeUtil;
incubator-eagle git commit: adjust description of dataSourceConfig.rMEndPoints in hadoop-queue-monitoring
Repository: incubator-eagle Updated Branches: refs/heads/master 66baf258c -> d4be077b8 adjust description of dataSourceConfig.rMEndPoints in hadoop-queue-monitoring Trivial modification, didn't open jira ticket for it. Author: anyway1021 <m...@apache.org> Closes #594 from anyway1021/adjust-desc. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/d4be077b Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/d4be077b Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/d4be077b Branch: refs/heads/master Commit: d4be077b892b62ac614908fb8f129d56efb2b191 Parents: 66baf25 Author: anyway1021 <m...@apache.org> Authored: Mon Nov 7 14:53:58 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Nov 7 14:53:58 2016 +0800 -- ...g.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d4be077b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml -- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml index a9ef355..7d614dc 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml +++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml @@ -24,8 +24,8 @@ dataSourceConfig.rMEndPoints - Resource Manager End Point - end point of resource manager, e.g. http://sandbox.hortonworks.com:8088/ + Resource Manager End Points + end points of resource manager, comma-separated for multiple http://sandbox.hortonworks.com:8088/ true
incubator-eagle git commit: [EAGLE-717] Upgrade maven-surefire-plugin for supporting forkMode ( =…
Repository: incubator-eagle Updated Branches: refs/heads/master f108c7f45 -> e39c76ce8 [EAGLE-717] Upgrade maven-surefire-plugin for supporting forkMode ( =⦠Be sure to do all of the following to help us incorporate your contribution quickly and easily: - [x] Make sure the PR title is formatted like: `[EAGLE-] Description of pull request` - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable Travis-CI on your fork and ensure the whole test matrix passes). - [ ] Replace `` in the title with the actual Jira issue number, if there is one. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt). --- ⦠always ) correctly Upgrade maven-surefire-plugin to 2.19.1 (latest version) Author: baibaichen <baibaic...@gmail.com> Closes #596 from baibaichen/EAGLE-717. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e39c76ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e39c76ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e39c76ce Branch: refs/heads/master Commit: e39c76ce8b503fcf447814cd9ea51c9ced2fee81 Parents: f108c7f Author: baibaichen <baibaic...@gmail.com> Authored: Thu Nov 3 15:59:51 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Nov 3 15:59:51 2016 +0800 -- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e39c76ce/pom.xml -- diff --git a/pom.xml b/pom.xml index 63f39d6..6792e4a 100755 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ 2.10 1.4.0 2.6 -2.6 +2.19.1 2.6 2.15.2 1.0 @@ -,6 +,7 @@ ${maven-surefire.version} -XX:-UseGCOverheadLimit +always
incubator-eagle git commit: [EAGLE-690] integrage topology health check with alert engine
Repository: incubator-eagle Updated Branches: refs/heads/master 2bc25969a -> f108c7f45 [EAGLE-690] integrage topology health check with alert engine integrage topology health check with alert engine Author: yupu <y...@ebay.com> Closes #570 from puyulu/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f108c7f4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f108c7f4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f108c7f4 Branch: refs/heads/master Commit: f108c7f456a02d92c70479895f61d3ab724745f8 Parents: 2bc2596 Author: yupu <y...@ebay.com> Authored: Thu Nov 3 14:47:28 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Nov 3 14:47:28 2016 +0800 -- .../apache/eagle/topology/TopologyCheckApp.java | 17 +++ .../eagle/topology/TopologyCheckAppConfig.java | 8 +- .../topology/extractor/TopologyCrawler.java | 3 +- .../extractor/TopologyEntityParser.java | 6 +- .../extractor/TopologyEntityParserResult.java | 2 +- .../hdfs/HdfsTopologyEntityParser.java | 14 +- .../topology/extractor/mr/YarnNodeInfo.java | 21 ++- .../extractor/mr/YarnNodeInfoWrapper.java | 2 +- .../topology/extractor/mr/YarnNodeInfos.java| 2 +- .../topology/resolver/TopologyRackResolver.java | 2 +- .../impl/IPMaskTopologyRackResolver.java| 10 +- .../topology/storm/HealthCheckParseBolt.java| 73 ++ .../topology/storm/TopologyDataPersistBolt.java | 72 +- .../topology/utils/EntityBuilderHelper.java | 18 +-- .../eagle/topology/utils/JMXQueryHelper.java| 6 +- .../utils/ServiceNotResponseException.java | 11 +- .../eagle/topology/utils/StringUtils.java | 45 ++ eagle.topology.TopologyCheckAppProvider.xml | 142 --- .../src/main/resources/application.conf | 33 +++-- .../entity/HBaseServiceTopologyAPIEntity.java | 4 +- .../entity/HdfsServiceTopologyAPIEntity.java| 11 +- .../entity/HealthCheckParseAPIEntity.java | 67 + .../entity/JournalNodeServiceAPIEntity.java | 4 +- .../entity/MRServiceTopologyAPIEntity.java | 11 +- .../entity/TopologyEntityRepository.java| 2 +- eagle-topology-check/pom.xml| 12 ++ 26 files changed, 474 insertions(+), 124 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f108c7f4/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java -- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java index 95bcb4d..93a06f8 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApp.java @@ -23,16 +23,25 @@ import backtype.storm.topology.TopologyBuilder; import com.typesafe.config.Config; import org.apache.eagle.app.StormApplication; import org.apache.eagle.app.environment.impl.StormEnvironment; +import org.apache.eagle.app.sink.StormStreamSink; +import org.apache.eagle.topology.storm.HealthCheckParseBolt; import org.apache.eagle.topology.storm.TopologyCheckAppSpout; import org.apache.eagle.topology.storm.TopologyDataPersistBolt; public class TopologyCheckApp extends StormApplication { + +private static final String SINK_TASK_NUM = "topology.numOfSinkTasks"; +private static final String TOPOLOGY_HEALTH_CHECK_STREAM = "topology_health_check_stream"; + @Override public StormTopology execute(Config config, StormEnvironment environment) { TopologyCheckAppConfig topologyCheckAppConfig = TopologyCheckAppConfig.newInstance(config); String spoutName = TopologyCheckAppConfig.TOPOLOGY_DATA_FETCH_SPOUT_NAME; String persistBoltName = TopologyCheckAppConfig.TOPOLOGY_ENTITY_PERSIST_BOLT_NAME; +String parseBoltName = TopologyCheckAppConfig.PARSE_BOLT_NAME; +String kafkaSinkBoltName = TopologyCheckAppConfig.SINK_BOLT_NAME; +int numOfSinkTasks = config.getInt(SINK_TASK_NUM); TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout( @@ -47,6 +56,14 @@ public class TopologyCheckApp extends StormApplication { topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt ).setNumTasks(topologyCheckAppConfig.dataExtractorConfig.numEntityPersistBolt).shuffleGrouping(spoutName); +topologyBu
incubator-eagle git commit: [EAGLE-707] Update JPM logic & ace editor support
Repository: incubator-eagle Updated Branches: refs/heads/master 8abbd2330 -> 0cd117c1f [EAGLE-707] Update JPM logic & ace editor support * modify jpm text content * update grunt script for ace editor support Author: zombieJ <smith3...@gmail.com> Closes #597 from zombieJ/EAGLE-707. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0cd117c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0cd117c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0cd117c1 Branch: refs/heads/master Commit: 0cd117c1fb798b6deda7c10bc72bd1e22dfd4da4 Parents: 8abbd23 Author: zombieJ <smith3...@gmail.com> Authored: Wed Nov 2 16:51:01 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 2 16:51:01 2016 +0800 -- .../src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js| 2 +- eagle-server/src/main/webapp/app/Gruntfile.js | 6 ++ eagle-server/src/main/webapp/app/build/index.js | 12 +++- eagle-server/src/main/webapp/app/dev/index.html | 5 - 4 files changed, 22 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0cd117c1/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js -- diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js index fc70bd3..67e2eab 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/jobTaskCtrl.js @@ -101,7 +101,7 @@ var startTime = Number($wrapState.param.startTime); var endTime = Number($wrapState.param.endTime); - PageConfig.title = "(beta) Task Statistic"; + PageConfig.title = "Task Statistic (Beta)"; PageConfig.subTitle = $scope.jobId; var timeDiff = endTime - startTime; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0cd117c1/eagle-server/src/main/webapp/app/Gruntfile.js -- diff --git a/eagle-server/src/main/webapp/app/Gruntfile.js b/eagle-server/src/main/webapp/app/Gruntfile.js index 3606d84..ecb94ef 100644 --- a/eagle-server/src/main/webapp/app/Gruntfile.js +++ b/eagle-server/src/main/webapp/app/Gruntfile.js @@ -75,6 +75,11 @@ module.exports = function (grunt) { {expand: true, cwd: 'dev/', src: '<%= config.copy.js.worker %>', dest: 'tmp'} ] }, + reference: { + files: [ + {expand: true, flatten: true, src: '<%= config.copy.js.reference %>', dest: 'tmp/public/js/reference'} + ] + }, ui: { files: [ {expand: true, cwd: 'tmp/', src: ['**'], dest: 'ui'}, @@ -172,6 +177,7 @@ module.exports = function (grunt) { // Compress JS 'concat:js_require', 'copy:worker', + 'copy:reference', 'concat:js_project', 'regex-replace:strict', 'uglify', http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0cd117c1/eagle-server/src/main/webapp/app/build/index.js -- diff --git a/eagle-server/src/main/webapp/app/build/index.js b/eagle-server/src/main/webapp/app/build/index.js index 2d274be..13c9aaf 100644 --- a/eagle-server/src/main/webapp/app/build/index.js +++ b/eagle-server/src/main/webapp/app/build/index.js @@ -62,6 +62,11 @@ var cssList = getResList('link[href][rel="stylesheet"]', 'href'); var jsList = getResList('script[src]', 'src'); + // Filter requirement + jsList.requireList = $.grep(jsList.requireList, function(src) { + return !/ace\.js/.test(src); + }); + // JS Worker process var workerFolderPath = 'dev/public/js/worker/'; var workerList = fs.readdirSync(workerFolderPath); @@ -120,7 +125,12 @@ }, copy: { js: { -
incubator-eagle git commit: [EAGLE-680] Can't genetate Html coverage report
Repository: incubator-eagle Updated Branches: refs/heads/master 77524c7c4 -> 8abbd2330 [EAGLE-680] Can't genetate Html coverage report EAGLE-680 Can't genetate Html coverage report - remove xml format in eagle-parent pom.xml Author: chitin <chitin1...@gmail.com> Closes #559 from chitin/EAGLE680. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8abbd233 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8abbd233 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8abbd233 Branch: refs/heads/master Commit: 8abbd23304bfed5c1a402353838eae9d511f84a7 Parents: 77524c7 Author: chitin <chitin1...@gmail.com> Authored: Wed Nov 2 11:32:18 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Nov 2 11:32:18 2016 +0800 -- pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8abbd233/pom.xml -- diff --git a/pom.xml b/pom.xml index 8f2ef44..63f39d6 100755 --- a/pom.xml +++ b/pom.xml @@ -1150,7 +1150,6 @@ cobertura-maven-plugin ${cobertura-maven.version} -xml 256m true
incubator-eagle git commit: [EAGLE-650] Hadoop JMX Metric Monitoring Application
Repository: incubator-eagle Updated Branches: refs/heads/master 3ff3e55ac -> 698b30282 [EAGLE-650] Hadoop JMX Metric Monitoring Application Hadoop JMX Metric Monitoring Application https://issues.apache.org/jira/browse/EAGLE-650 Monitored Services: * hdfs name node * yarn resource manager node * hbase master node Author: Hao Chen <h...@apache.org> Closes #552 from haoch/EAGLE-650. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/698b3028 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/698b3028 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/698b3028 Branch: refs/heads/master Commit: 698b302821c7a89cee1172dbf8d2cdb296506b54 Parents: 3ff3e55 Author: Hao Chen <h...@apache.org> Authored: Mon Oct 24 19:37:58 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Oct 24 19:37:58 2016 +0800 -- ...e.alert.app.AlertUnitTopologyAppProvider.xml | 3 +- .../app/environment/impl/StaticEnvironment.java | 19 +--- .../eagle/app/service/ApplicationAction.java| 14 +-- .../apache/eagle/app/sink/KafkaStreamSink.java | 39 ++-- .../app/environment/StaticEnvironmentTest.java | 10 ++- .../apache/eagle/metadata/model/StreamDesc.java | 3 +- .../metadata/utils/StreamIdConversions.java | 46 ++ .../metadata/utils/StreamIdConversionsTest.java | 37 ...e.app.example.ExampleApplicationProvider.xml | 3 +- ...apache.eagle.gc.GCLogApplicationProvider.xml | 5 +- eagle-hadoop-metric/pom.xml | 6 +- .../metric/HadoopMetricMonitorAppProdiver.java | 23 + ...le.metric.HadoopMetricMonitorAppProdiver.xml | 94 ...org.apache.eagle.app.spi.ApplicationProvider | 16 .../HadoopMetricMonitorAppProdiverTest.java | 84 + ...doop.queue.HadoopQueueRunningAppProvider.xml | 2 +- ...gregation.AggregationApplicationProvider.xml | 2 +- security.hbase.HBaseAuditLogAppProvider.xml | 4 +- ...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 2 +- eagle-server/pom.xml| 7 ++ 20 files changed, 370 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index 28f7db4..7b12917 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -18,7 +18,8 @@ AlertUnitTopologyApp -Alert Unit Topology Application +Alert Engine +Real-time Alert Engine 0.5.0-incubating http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java index 1c297fb..1c17b1f 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java @@ -17,23 +17,10 @@ package org.apache.eagle.app.environment.impl; import com.typesafe.config.Config; -import org.apache.eagle.app.environment.Environment; -import org.apache.eagle.app.sink.StreamSinkProvider; - -public class StaticEnvironment implements Environment { -private final Config config; +import org.apache.eagle.app.environment.AbstractEnvironment; +public class StaticEnvironment extends AbstractEnvironment { public StaticEnvironment(Config config) { -this.config = config; -} - -@Override -public Config config() { -return this.config; -} - -@Override -public StreamSinkProvider streamSink() { -throw new IllegalStateException("streamSink() is not supported in " + StaticEnvironment.class.getSimpl
incubator-eagle git commit: [MINOR] Register StormMetricTaggedConsumer only when metric.sink exists
Repository: incubator-eagle Updated Branches: refs/heads/master 435451eff -> 8bd30aec1 [MINOR] Register StormMetricTaggedConsumer only when metric.sink exists Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8bd30aec Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8bd30aec Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8bd30aec Branch: refs/heads/master Commit: 8bd30aec1e75f964a0e45af605f37e1b672ba312 Parents: 435451e Author: Hao Chen <h...@apache.org> Authored: Sun Oct 23 21:04:33 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Sun Oct 23 21:04:33 2016 +0800 -- .../apache/eagle/app/environment/impl/StormExecutionRuntime.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8bd30aec/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java index ccfcd93..4a9b2b7 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java @@ -23,6 +23,7 @@ import backtype.storm.utils.NimbusClient; import com.google.common.base.Preconditions; import com.typesafe.config.ConfigRenderOptions; import org.apache.eagle.alert.engine.runner.StormMetricTaggedConsumer; +import org.apache.eagle.alert.metric.MetricConfigs; import org.apache.eagle.app.Application; import org.apache.eagle.app.environment.ExecutionRuntime; import org.apache.eagle.app.environment.ExecutionRuntimeProvider; @@ -61,7 +62,6 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment, } public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs"; -public static final String METRIC_CONFIG_PREFIX = "metric"; private static final String STORM_NIMBUS_HOST_CONF_PATH = "application.storm.nimbusHost"; private static final String STORM_NIMBUS_HOST_DEFAULT = "localhost"; @@ -103,7 +103,7 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment, conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, config.getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS)); } -if (config.hasPath(METRIC_CONFIG_PREFIX)) { +if (config.hasPath(MetricConfigs.METRIC_SINK_CONF)) { conf.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1); } return conf;
incubator-eagle git commit: Minor: Expose several async producer config
Repository: incubator-eagle Updated Branches: refs/heads/master 281aa52b0 -> 3980dcb4c Minor: Expose several async producer config Author: Zhao, Qingwen <qingwz...@apache.org> Closes #549 from qingwen220/kafkaProducer. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3980dcb4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3980dcb4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3980dcb4 Branch: refs/heads/master Commit: 3980dcb4c34f024e8247747a48f5966438415c82 Parents: 281aa52 Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Fri Oct 21 22:27:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Fri Oct 21 22:27:55 2016 +0800 -- .../apache/eagle/app/sink/KafkaStreamSink.java | 10 ++ .../eagle/app/sink/KafkaStreamSinkConfig.java | 36 ...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 25 ++ .../src/main/resources/application.conf | 4 +++ 4 files changed, 75 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java index 2ac4779..e2a4b70 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java @@ -49,6 +49,11 @@ public class KafkaStreamSink extends StormStreamSink { properties.put("metadata.broker.list", config.getBrokerList()); properties.put("serializer.class", config.getSerializerClass()); properties.put("key.serializer.class", config.getKeySerializerClass()); +// new added properties for async producer +properties.put("producer.type", config.getProducerType()); +properties.put("batch.num.messages", config.getNumBatchMessages()); +properties.put("request.required.acks", config.getRequestRequiredAcks()); +properties.put("queue.buffering.max.ms", config.getMaxQueueBufferMs()); ProducerConfig producerConfig = new ProducerConfig(properties); producer = new Producer(producerConfig); } @@ -97,6 +102,11 @@ public class KafkaStreamSink extends StormStreamSink { desc.setBrokerList(config.getString("dataSinkConfig.brokerList")); desc.setSerializerClass(config.getString("dataSinkConfig.serializerClass")); desc.setKeySerializerClass(config.getString("dataSinkConfig.keySerializerClass")); +// new added properties for async producer + desc.setNumBatchMessages(config.getString("dataSinkConfig.numBatchMessages")); + desc.setProducerType(config.getString("dataSinkConfig.producerType")); + desc.setMaxQueueBufferMs(config.getString("dataSinkConfig.maxQueueBufferMs")); + desc.setRequestRequiredAcks(config.getString("dataSinkConfig.requestRequiredAcks")); return desc; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java index 9d6a0ab..d5479df 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java @@ -23,6 +23,10 @@ public class KafkaStreamSinkConfig implements StreamSinkConfig { private String brokerList; private String serializerClass; private String keySerializerClass; +private String numBatchMessages; +private String maxQueueBufferMs; +private String producerType; +private String requestRequiredAcks; public String getTopicId() { return topicId; @@ -56,6 +60,38 @@ public class KafkaStreamSinkConfig implements StreamSinkConfig { this.keySerializerClass = keySerializerClass; } +public String getNumBatchMessages
incubator-eagle git commit: [EAGLE-652] add topology health check to app framework
Repository: incubator-eagle Updated Branches: refs/heads/master 4472fa1f5 -> eafff46c3 [EAGLE-652] add topology health check to app framework add topology health check to app framework Author: yupu <y...@ebay.com> Closes #546 from puyulu/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/eafff46c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/eafff46c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/eafff46c Branch: refs/heads/master Commit: eafff46c3df9e245120e5d7235a040d06a3715ce Parents: 4472fa1 Author: yupu <y...@ebay.com> Authored: Fri Oct 21 12:23:49 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Fri Oct 21 12:23:49 2016 +0800 -- eagle-server/pom.xml | 7 +++ .../services/org.apache.eagle.app.spi.ApplicationProvider | 1 + .../topology/extractor/hdfs/HdfsTopologyEntityParser.java | 10 ++ .../apache/eagle/topology/utils/EntityBuilderHelper.java | 8 ...org.apache.eagle.topology.TopologyCheckAppProvider.xml | 2 -- .../java/org/apache/eagle/topology/TopologyConstants.java | 2 ++ 6 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eafff46c/eagle-server/pom.xml -- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index 88df514..1a3e7c9 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -321,6 +321,13 @@ eagle-hadoop-queue ${project.version} + + + +org.apache.eagle +eagle-topology-app +${project.version} + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eafff46c/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider -- diff --git a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider index 86e5e0a..4a3dc12 100644 --- a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider +++ b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider @@ -25,3 +25,4 @@ org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider org.apache.eagle.jpm.aggregation.AggregationApplicationProvider org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider +org.apache.eagle.topology.TopologyCheckAppProvider \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eafff46c/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java -- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java index f2cad37..1cec474 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java @@ -78,6 +78,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { private static final String STATUS_PATTERN = "([\\d\\.]+):\\d+\\s+\\([\\D]+(\\d+)\\)"; private static final String QJM_PATTERN = "([\\d\\.]+):\\d+"; + +private static final double TB = 1024 * 1024 * 1024 * 1024; public HdfsTopologyEntityParser(String site, TopologyCheckAppConfig.HdfsConfig hdfsConfig, TopologyRackResolver rackResolver) { this.namenodeUrls = hdfsConfig.namenodeUrls; @@ -200,7 +202,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser { for (int i = 0; deadNodes != null && i < deadNodes.length(); ++i) { final String hostname = deadNodes.getString(i); final JSONObject deadNode = jsonNodesObject.getJSONObject(hostname); -HdfsServiceTopologyAPIEntity entity = createHdfsServiceEntity(TopologyConstants.DATA_NODE_ROLE, hostname, updateTime); +HdfsServic
incubator-eagle git commit: [EAGLE-656] Integrate metric system with application framework
Repository: incubator-eagle Updated Branches: refs/heads/master 49db82c5e -> 4472fa1f5 [EAGLE-656] Integrate metric system with application framework Integrate metric system with application framework https://issues.apache.org/jira/browse/EAGLE-656 Author: Hao Chen <h...@apache.org> Closes #544 from haoch/IntegrateMetricSystem. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4472fa1f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4472fa1f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4472fa1f Branch: refs/heads/master Commit: 4472fa1f5a5a664acc6b9f486d6fe5c46a3b01ac Parents: 49db82c Author: Hao Chen <h...@apache.org> Authored: Fri Oct 21 10:44:40 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Fri Oct 21 10:44:40 2016 +0800 -- .../eagle/alert/metric/MetricConfigs.java | 25 +++ .../apache/eagle/alert/metric/MetricSystem.java | 34 -- .../alert/metric/sink/ElasticSearchSink.java| 40 - .../eagle/alert/metric/sink/KafkaSink.java | 5 ++- .../src/test/resources/application.conf | 5 ++- .../engine/runner/StormMetricConsumer.java | 6 +-- .../runner/StormMetricTaggedConsumer.java | 47 +--- .../environment/impl/StormExecutionRuntime.java | 7 +++ .../eagle/app/service/ApplicationAction.java| 34 -- .../app/test/ApplicationSimulatorImpl.java | 11 - .../eagle/app/TestApplicationSimulatorImpl.java | 41 + 11 files changed, 196 insertions(+), 59 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4472fa1f/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricConfigs.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricConfigs.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricConfigs.java new file mode 100644 index 000..81aa75c --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricConfigs.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.alert.metric; + +public class MetricConfigs { +public static final String METRIC_PREFIX_CONF = "metric.prefix"; +public static final String METRIC_SINK_CONF = "metric.sink"; +public static final String DURATION_SECONDS_CONF = "metric.durationSeconds"; +public static final String TAGS_FIELD_NAME = "tags"; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4472fa1f/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java index 555c4ec..255f1e6 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java @@ -16,12 +16,12 @@ */ package org.apache.eagle.alert.metric; -import org.apache.eagle.alert.metric.sink.MetricSink; -import org.apache.eagle.alert.metric.sink.MetricSinkRepository; -import org.apache.eagle.alert.metric.source.MetricSource; import com.codahale.metrics.MetricRegistry; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import org.apache.eagle.alert.metric.sink.Met
incubator-eagle git commit: [MINOR] Resolve unstable UT and clean slf4j-log4j dependencies
Repository: incubator-eagle Updated Branches: refs/heads/master 64fce8f80 -> 605c8b21a [MINOR] Resolve unstable UT and clean slf4j-log4j dependencies Resolve unstable UT and clean useless `slf4j-log4j` dependencies. Author: Hao Chen <h...@apache.org> Closes #539 from haoch/ResolveUnstableUT. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/605c8b21 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/605c8b21 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/605c8b21 Branch: refs/heads/master Commit: 605c8b21afcecd5d09444755302bce34b3400c2b Parents: 64fce8f Author: Hao Chen <h...@apache.org> Authored: Thu Oct 20 17:21:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Oct 20 17:21:55 2016 +0800 -- .../eagle-embed/eagle-embed-server/pom.xml | 13 +- .../service/embedded/tomcat/EmbeddedServer.java | 129 +-- .../embedded/tomcat/TestEmbeddedServer.java | 56 3 files changed, 99 insertions(+), 99 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/605c8b21/eagle-core/eagle-embed/eagle-embed-server/pom.xml -- diff --git a/eagle-core/eagle-embed/eagle-embed-server/pom.xml b/eagle-core/eagle-embed/eagle-embed-server/pom.xml index a07ba59..4ef62a0 100644 --- a/eagle-core/eagle-embed/eagle-embed-server/pom.xml +++ b/eagle-core/eagle-embed/eagle-embed-server/pom.xml @@ -54,16 +54,13 @@ tomcat-embed-websocket -log4j -log4j +org.slf4j +slf4j-api - - - - -org.slf4j -log4j-over-slf4j +junit +junit +test http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/605c8b21/eagle-core/eagle-embed/eagle-embed-server/src/main/java/org/apache/eagle/service/embedded/tomcat/EmbeddedServer.java -- diff --git a/eagle-core/eagle-embed/eagle-embed-server/src/main/java/org/apache/eagle/service/embedded/tomcat/EmbeddedServer.java b/eagle-core/eagle-embed/eagle-embed-server/src/main/java/org/apache/eagle/service/embedded/tomcat/EmbeddedServer.java index 35d3967..203e52b 100644 --- a/eagle-core/eagle-embed/eagle-embed-server/src/main/java/org/apache/eagle/service/embedded/tomcat/EmbeddedServer.java +++ b/eagle-core/eagle-embed/eagle-embed-server/src/main/java/org/apache/eagle/service/embedded/tomcat/EmbeddedServer.java @@ -16,23 +16,23 @@ */ package org.apache.eagle.service.embedded.tomcat; -import java.io.File; - import org.apache.catalina.LifecycleState; import org.apache.catalina.startup.Tomcat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; + public class EmbeddedServer { - - private static EmbeddedServer server; - private Tomcat tomcat; - private String webappDirLocation; - private int port; - private static int DEFAULT_TOMCAT_PORT = 38080; - private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); - - public static void main(String[] args) { + +private static EmbeddedServer server; +private Tomcat tomcat; +private String webappDirLocation; +private int port; +private static int DEFAULT_TOMCAT_PORT = 38080; +private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); + +public static void main(String[] args) { // args: webappLocation, port int tomcatPort; if (args.length > 1) { @@ -41,69 +41,68 @@ public class EmbeddedServer { tomcatPort = DEFAULT_TOMCAT_PORT; } new EmbeddedServer(args[0], tomcatPort).start(); - } +} + +private EmbeddedServer(String webappDirLocation) { +this(webappDirLocation, DEFAULT_TOMCAT_PORT); +} + +private EmbeddedServer(String webappDirLocation, int port) { +this.webappDirLocation = webappDirLocation; +this.port = port; +} - private EmbeddedServer(String webappDirLocation) { - this(webappDirLocation, DEFAULT_TOMCAT_PORT); - } - - private EmbeddedServer(String webappDirLocation, int port) { - this.webappDirLocation = webappDirLocation; - this.port = port; - } - public static EmbeddedServer getInstance(String webappDirLocation) { - if (server == null) { - synchronized(EmbeddedServer.class) { - if (server == null) { - server = new Embedded
incubator-eagle git commit: [MINOR] Fix cobertura.totalLineRate as 25 and improve some UT
Repository: incubator-eagle Updated Branches: refs/heads/master 538e88f71 -> 49db82c5e [MINOR] Fix cobertura.totalLineRate as 25 and improve some UT [MINOR] Fix `cobertura.totalLineRate` as 25 and improve some unit test cases Author: Hao Chen <h...@apache.org> Closes #543 from haoch/FixCoberturaRatioAndAddUT. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/49db82c5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/49db82c5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/49db82c5 Branch: refs/heads/master Commit: 49db82c5e129c7dc1d0bd5932c97518891c6158e Parents: 538e88f Author: Hao Chen <h...@apache.org> Authored: Thu Oct 20 22:21:01 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Oct 20 22:21:01 2016 +0800 -- .../app/AlertUnitTopologyAppProviderTest.java | 69 .../src/test/resource/application.conf | 65 +++ .../eagle/alert/model/StreamEventTest.java | 68 .../correlation/meta/StreamPartitionTest.java | 36 + .../org/apache/eagle/app/StaticApplication.java | 10 +-- .../environment/ExecutionRuntimeManager.java| 2 +- .../app/environment/impl/StaticEnvironment.java | 39 + .../impl/StaticExecutionRuntime.java| 68 .../app/environment/impl/WebEnvironment.java| 39 - .../environment/impl/WebExecutionRuntime.java | 68 .../app/test/ApplicationSimulatorImpl.java | 6 +- .../app/environment/StaticEnvironmentTest.java | 35 + .../app/resource/ApplicationResourceTest.java | 83 .../eagle/common/TestSerializableUtils.java | 53 + pom.xml | 4 + 15 files changed, 528 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/49db82c5/eagle-core/eagle-alert-parent/eagle-alert-app/src/test/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProviderTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/test/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProviderTest.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/test/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProviderTest.java new file mode 100644 index 000..927d505 --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/test/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProviderTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.alert.app; + +import com.google.inject.Inject; +import org.apache.eagle.app.resource.ApplicationResource; +import org.apache.eagle.app.service.ApplicationOperations; +import org.apache.eagle.app.test.ApplicationTestBase; +import org.apache.eagle.metadata.model.ApplicationEntity; +import org.apache.eagle.metadata.model.SiteEntity; +import org.apache.eagle.metadata.resource.SiteResource; +import org.apache.eagle.metadata.service.ApplicationStatusUpdateService; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class AlertUnitTopologyAppProviderTest extends ApplicationTestBase { + +@Inject +private SiteResource siteResource; +@Inject +private ApplicationResource applicationResource; +@Inject +ApplicationStatusUpdateService statusUpdateService; + +@Test +public void testApplicationLifecycle() throws InterruptedException { +// Create local site +SiteEntity siteEntity = new SiteEntity(); +siteEntity.setSiteId("test_site"); +siteEntity.setSiteName("Test Site"); +siteEntity.setDescription("Test Site for ExampleApplicationProviderTest"); +siteResource.createSite(siteEntity); +Assert.assertNotNull(siteEntity.getUuid()); +
[2/2] incubator-eagle git commit: [EAGLE-647] Support Policy Execution Interpreter and Planner to compile siddhi query to distributed execution plan
[EAGLE-647] Support Policy Execution Interpreter and Planner to compile siddhi query to distributed execution plan Support Policy Execution Interpreter and Planner to compile siddhi query to distributed execution plan * Support parse siddhi pattern and join query as distributed execution * Support alias in inner join condition * Refactor PolicyIntepreter to eagle-alert-engine and decoupel PolicyExecutionPlanner * Fix factory method for PolicyExecutionPlanner Author: Hao Chen <h...@apache.org> Closes #536 from haoch/SupportPatternAndJoin. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/64fce8f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/64fce8f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/64fce8f8 Branch: refs/heads/master Commit: 64fce8f80745d6180ff04b6950364a802d73723e Parents: 8991b61 Author: Hao Chen <h...@apache.org> Authored: Thu Oct 20 16:08:33 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Thu Oct 20 16:08:33 2016 +0800 -- .../engine/interpreter/PolicyExecutionPlan.java | 100 + .../interpreter/PolicyExecutionPlanner.java | 28 ++ .../interpreter/PolicyExecutionPlannerImpl.java | 339 + .../engine/interpreter/PolicyInterpreter.java | 112 ++ .../engine/interpreter/PolicyParseResult.java | 65 .../interpreter/PolicyValidationResult.java | 76 .../interpreter/PolicyInterpreterTest.java | 381 +++ .../metadata/resource/MetadataResource.java | 9 +- .../metadata/resource/PolicyExecutionPlan.java | 100 - .../metadata/resource/PolicyInterpreter.java| 244 .../metadata/resource/PolicyParseResult.java| 65 .../resource/PolicyValidationResult.java| 76 .../resource/PolicyInterpreterTest.java | 226 --- 13 files changed, 1109 insertions(+), 712 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/64fce8f8/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlan.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlan.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlan.java new file mode 100644 index 000..7ecc36f --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/interpreter/PolicyExecutionPlan.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.alert.engine.interpreter; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.eagle.alert.engine.coordinator.StreamColumn; +import org.apache.eagle.alert.engine.coordinator.StreamPartition; +import org.wso2.siddhi.query.api.ExecutionPlan; + +import java.util.List; +import java.util.Map; + +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PolicyExecutionPlan { +/** + * Actual input streams. + */ +private Map<String, List> inputStreams; + +/** + * Actual output streams. + */ +private Map<String, List> outputStreams; + +/** + * Execution plan source. + */ +private String executionPlanSource; + +/** + * Execution plan. + */ +private ExecutionPlan internalExecutionPlan; + +private String executionPlanDesc; + +private List streamPartitions; + +public String getExecutionPlanSource() { +return executionPlanSource; +} + +public void setExecutionPlanSource(String executionPlanSource) { +this.executionPlanSource = executionPlanSource; +} + +public ExecutionPlan getInternalExecutionPlan() { +retur
[1/2] incubator-eagle git commit: [EAGLE-647] Support Policy Execution Interpreter and Planner to compile siddhi query to distributed execution plan
Repository: incubator-eagle Updated Branches: refs/heads/master 8991b61e2 -> 64fce8f80 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/64fce8f8/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/PolicyValidationResult.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/PolicyValidationResult.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/PolicyValidationResult.java deleted file mode 100644 index b1912af..000 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/PolicyValidationResult.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.metadata.resource; - - -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.eagle.alert.engine.coordinator.PolicyDefinition; - -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class PolicyValidationResult { -private boolean success; -private String message; -private String exception; - -private PolicyExecutionPlan policyExecutionPlan; -private PolicyDefinition policyDefinition; - -public String getException() { -return exception; -} - -public void setException(String exception) { -this.exception = exception; -} - -public String getMessage() { -return message; -} - -public void setMessage(String message) { -this.message = message; -} - -public void setStackTrace(Throwable throwable) { -this.setException(ExceptionUtils.getStackTrace(throwable)); -} - -public boolean isSuccess() { -return success; -} - -public void setSuccess(boolean success) { -this.success = success; -} - -public PolicyExecutionPlan getPolicyExecutionPlan() { -return policyExecutionPlan; -} - -public void setPolicyExecutionPlan(PolicyExecutionPlan policyExecutionPlan) { -this.policyExecutionPlan = policyExecutionPlan; -} - -public PolicyDefinition getPolicyDefinition() { -return policyDefinition; -} - -public void setPolicyDefinition(PolicyDefinition policyDefinition) { -this.policyDefinition = policyDefinition; -} -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/64fce8f8/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/metadata/resource/PolicyInterpreterTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/metadata/resource/PolicyInterpreterTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/metadata/resource/PolicyInterpreterTest.java deleted file mode 100644 index 48ac9eb..000 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/metadata/resource/PolicyInterpreterTest.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software
incubator-eagle git commit: [EAGLE-639] Generate sortSpec only on externalWindow
Repository: incubator-eagle Updated Branches: refs/heads/master 453c3a5fa -> 6f88c30f8 [EAGLE-639] Generate sortSpec only on externalWindow Generate sortSpec on externalWindow https://issues.apache.org/jira/browse/EAGLE-639 Author: Hao Chen <h...@apache.org> Closes #532 from haoch/EnhancePolicyIntepreter. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6f88c30f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6f88c30f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6f88c30f Branch: refs/heads/master Commit: 6f88c30f855249c401a40efc6941090a152670f3 Parents: 453c3a5 Author: Hao Chen <h...@apache.org> Authored: Wed Oct 19 17:15:06 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Oct 19 17:15:06 2016 +0800 -- .../engine/sorter/StreamSortHandlerTest.java| 7 +- .../metadata/resource/MetadataResource.java | 4 +- .../metadata/resource/PolicyCompiler.java | 235 -- .../metadata/resource/PolicyInterpreter.java| 244 +++ .../metadata/resource/PolicyCompilerTest.java | 195 --- .../resource/PolicyInterpreterTest.java | 226 + 6 files changed, 476 insertions(+), 435 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6f88c30f/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java index 98657e7..885017a 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java @@ -32,6 +32,7 @@ import org.apache.eagle.alert.engine.sorter.impl.StreamTimeClockInLocalMemory; import org.apache.eagle.alert.utils.DateTimeUtil; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,7 +136,7 @@ public class StreamSortHandlerTest { * * @throws InterruptedException */ -@Test +@Test @Ignore("Igored heavy benchmark test in unit test") public void testWithUnsortedEventsInLargeWindowBenchmark() throws InterruptedException { metricReporter.report(); testWithUnsortedEventsIn1hWindow(1000); @@ -146,8 +147,8 @@ public class StreamSortHandlerTest { metricReporter.report(); testWithUnsortedEventsIn1hWindow(100); metricReporter.report(); -//testWithUnsortedEventsIn1hWindow(1000); -//metricReporter.report(); +testWithUnsortedEventsIn1hWindow(1000); +metricReporter.report(); } public void testWithUnsortedEventsIn1hWindow(int count) throws InterruptedException { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6f88c30f/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java index eb23362..190da2a 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java @@ -207,13 +207,13 @@ public class MetadataResource { @Path("/policies/validate") @POST public PolicyValidationResult validatePolicy(PolicyDefinition policy) { -return PolicyCompiler.validate(policy,dao); +return PolicyInterpreter.validate(policy,dao); } @Path("/policies/parse") @POST public PolicyParseResult parsePolicy(String policyDefinition) { -return Poli
[50/50] incubator-eagle git commit: [EAGLE-635] Refactor policy parser and validator for richer plan details and better performance
[EAGLE-635] Refactor policy parser and validator for richer plan details and better performance ## Changes * Refactor policy parser and validator for richer plan details and better performance * Decouple PolicyExecutionPlan and PolicyValidation ## API * Parse API ~~~ POST /metadata/policies/parse Accept-Type: text from HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX#window.timeBatch(2 min) select cmd, user, count() as total_count group by cmd,user insert into HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT" ~~~ * Validation API ~~~ POST /metadata/policies/validate Accept-Type: application/json { "name": "hdfsPolicy", "description": "hdfsPolicy", "inputStreams": [ "HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX" ], "outputStreams": [ "HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT" ], "definition": { "type": "siddhi", "value": "from HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX#window.timeBatch(2 min) select cmd, user, count() as total_count group by cmd,user insert into HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT " }, "partitionSpec": [ { "streamId": "hdfs_audit_log_enriched_stream", "type": "GROUPBY", "columns" : [ "cmd" ] } ], "parallelismHint": 2 } ~~~ Author: Hao Chen <h...@apache.org> Closes #529 from haoch/RefactorPolicyValidator. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6dbdb4f7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6dbdb4f7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6dbdb4f7 Branch: refs/heads/master Commit: 6dbdb4f72ec5adf89f4f7a13bf766e01e3ff6705 Parents: adc2ba9 Author: Hao Chen <h...@apache.org> Authored: Wed Oct 19 10:28:09 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Oct 19 10:28:09 2016 +0800 -- .../conf/sandbox-userprofile-scheduler.conf | 2 +- .../eagle-alert/alert-common/pom.xml| 8 - .../eagle-alert/alert-engine/pom.xml| 10 +- .../evaluator/impl/SiddhiDefinitionAdapter.java | 13 + .../alert-metadata-service/pom.xml | 13 - .../metadata/resource/MetadataResource.java | 10 +- .../metadata/resource/PolicyCompiler.java | 235 +++ .../metadata/resource/PolicyExecutionPlan.java | 100 .../metadata/resource/PolicyParseResult.java| 65 + .../metadata/resource/PolicyValidation.java | 97 .../resource/PolicyValidationResult.java| 76 ++ .../metadata/resource/PolicyValidator.java | 124 -- .../metadata/resource/PolicyCompilerTest.java | 195 +++ .../metadata/resource/PolicyValidatorTest.java | 187 --- 14 files changed, 699 insertions(+), 436 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6dbdb4f7/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf -- diff --git a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf b/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf index f54162d..4593a7e 100644 --- a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf +++ b/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf @@ -63,4 +63,4 @@ akka { # Filter of log events that is used by the LoggingAdapter before # publishing log events to the eventStream. logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6dbdb4f7/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml index 57a0157..fd3eccd 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml @@ -63,10 +63,6 @@ slf4j-api -org.slf4j -slf4j-log4j12 - - org.apache.curator curator-framework @@ -82,10 +78,6 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - - - joda-time joda-time http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6dbdb4f7/eagle-core/eagle-alert-p
[07/50] incubator-eagle git commit: [MINOR] Update HdfsAuditLogAppProvider.xml
[MINOR] Update HdfsAuditLogAppProvider.xml Author: Zhao, QingwenCloses #488 from qingwen220/hdfsAuditLog. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9205433b Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9205433b Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9205433b Branch: refs/heads/master Commit: 9205433bccefd1b8e477dc09ef7e7ef5ae15ab41 Parents: 36df3ee Author: Zhao, Qingwen Authored: Tue Oct 11 09:55:14 2016 +0800 Committer: Zhao, Qingwen Committed: Tue Oct 11 09:55:14 2016 +0800 -- .../environment/impl/StormExecutionRuntime.java | 2 +- security.auditlog.HdfsAuditLogAppProvider.xml | 18 +++--- .../src/main/resources/application.conf | 1 - 3 files changed, 16 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9205433b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java index 9821660..1994e28 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java @@ -157,7 +157,7 @@ public class StormExecutionRuntime implements ExecutionRuntime executor, com.typesafe.config.Config config) { String appId = config.getString("appId"); -LOG.info("Fetching status of topology {} ..." + appId); +LOG.info("Fetching status of topology {} ...", appId); List topologySummaries ; try { if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9205433b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml -- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml index cd564df..625a3a7 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml @@ -27,6 +27,12 @@ +workers +workers +1 +number of topology workers + + topology.numOfSpoutTasks topology.numOfSpoutTasks 2 @@ -39,10 +45,16 @@ number of parser tasks -topology.numOfJoinTasks -topology.numOfJoinTasks +topology.numOfSensitivityJoinTasks +topology.numOfSensitivityJoinTasks +2 +number of sensitivity join tasks + + +topology.numOfIPZoneJoinTasks +topology.numOfIPZoneJoinTasks 2 -number of external join tasks +number of ip zone join tasks topology.numOfSinkTasks http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9205433b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf -- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf index ed4609d..c8bbcb1 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf @@ -18,7 +18,6 @@ "mode" : "LOCAL", "siteId" : "testsite", "topology" : { -"numOfTotalWorkers" : 2, "numOfSpoutTasks" : 2, "numOfParserTasks" : 2,
[05/50] incubator-eagle git commit: [MINOR] Support PUT in Access-Control-Allow-Methods
[MINOR] Support PUT in Access-Control-Allow-Methods Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7d502533 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7d502533 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7d502533 Branch: refs/heads/master Commit: 7d5025336b647ed7afc21e6ddbbeaac2c074ea36 Parents: 4b53793 Author: Hao Chen <h...@apache.org> Authored: Mon Oct 10 19:19:35 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Oct 10 19:19:35 2016 +0800 -- .../java/org/apache/eagle/alert/resource/SimpleCORSFiler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7d502533/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java index 2e1c72c..5b8141c 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java @@ -44,15 +44,15 @@ public class SimpleCORSFiler implements Filter { ServletException { HttpServletResponse response = (HttpServletResponse) res; response.setHeader("Access-Control-Allow-Origin", "*"); -response.setHeader("Access-Control-Allow-Methods", "HEAD, POST, GET, OPTIONS, DELETE"); +response.setHeader("Access-Control-Allow-Methods", "HEAD, POST, GET, PUT, OPTIONS, DELETE"); response.setHeader("Access-Control-Max-Age", "3600"); response.setHeader("Access-Control-Allow-Headers", -"Origin, Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers"); +"Origin, Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers"); chain.doFilter(request, response); } @Override public void destroy() { -} +} }
[21/50] incubator-eagle git commit: [MINOR] Update unitApp configuration
[MINOR] Update unitApp configuration 1. remove 'topology.name' in AlertUnitTopologyAppProvider.xml 2. remove topology assembly in jpm Author: Zhao, QingwenCloses #502 from qingwen220/unitApp. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/5e57c7bb Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/5e57c7bb Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/5e57c7bb Branch: refs/heads/master Commit: 5e57c7bbc81bf2603ed56ccb5428aeb9869fa48d Parents: 806a29c Author: Zhao, Qingwen Authored: Fri Oct 14 09:46:49 2016 +0800 Committer: Zhao, Qingwen Committed: Fri Oct 14 09:46:49 2016 +0800 -- ...e.alert.app.AlertUnitTopologyAppProvider.xml | 6 -- eagle-jpm/eagle-hadoop-queue/pom.xml| 20 -- .../assembly/eagle-running-queue-assembly.xml | 66 eagle-jpm/eagle-jpm-aggregation/pom.xml | 20 -- .../assembly/eagle-jpm-aggregation-assembly.xml | 65 --- eagle-jpm/eagle-jpm-mr-history/pom.xml | 20 -- .../assembly/eagle-jpm-mr-history-assembly.xml | 65 --- eagle-jpm/eagle-jpm-mr-running/pom.xml | 20 -- .../assembly/eagle-jpm-mr-running-assembly.xml | 65 --- eagle-jpm/eagle-jpm-spark-history/pom.xml | 20 -- .../eagle-jpm-spark-history-assembly.xml| 65 --- eagle-jpm/eagle-jpm-spark-running/pom.xml | 20 -- .../eagle-jpm-spark-running-assembly.xml| 65 --- 13 files changed, 517 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5e57c7bb/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index 0188210..bf22123 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -25,12 +25,6 @@ -topology.name -topology.name -alertUnitTopology_1 -alert topology name - - topology.numOfTotalWorkers topology.numOfTotalWorkers 2 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5e57c7bb/eagle-jpm/eagle-hadoop-queue/pom.xml -- diff --git a/eagle-jpm/eagle-hadoop-queue/pom.xml b/eagle-jpm/eagle-hadoop-queue/pom.xml index f9cfed5..91568ff 100644 --- a/eagle-jpm/eagle-hadoop-queue/pom.xml +++ b/eagle-jpm/eagle-hadoop-queue/pom.xml @@ -65,26 +65,6 @@ src/main/resources - - -maven-assembly-plugin - - src/assembly/eagle-running-queue-assembly.xml - eagle-running-queue-${project.version} - - - -package - -single - - -posix - - - - - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/5e57c7bb/eagle-jpm/eagle-hadoop-queue/src/assembly/eagle-running-queue-assembly.xml -- diff --git a/eagle-jpm/eagle-hadoop-queue/src/assembly/eagle-running-queue-assembly.xml b/eagle-jpm/eagle-hadoop-queue/src/assembly/eagle-running-queue-assembly.xml deleted file mode 100644 index 244c5f0..000 --- a/eagle-jpm/eagle-hadoop-queue/src/assembly/eagle-running-queue-assembly.xml +++ /dev/null @@ -1,66 +0,0 @@ - - -http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd;> -assembly - -jar - -false - - -/ -false -
[37/50] incubator-eagle git commit: [MINOR] Add missed license header in BasicAuthProviderBuilder
[MINOR] Add missed license header in BasicAuthProviderBuilder Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e8a73893 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e8a73893 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e8a73893 Branch: refs/heads/master Commit: e8a73893d5d395fdba3716717f728e8768ae Parents: b103e20 Author: Hao Chen <h...@apache.org> Authored: Mon Oct 17 11:10:16 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Oct 17 11:10:16 2016 +0800 -- .../authentication/BasicAuthProviderBuilder.java| 16 1 file changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e8a73893/eagle-server/src/main/java/org/apache/eagle/server/authentication/BasicAuthProviderBuilder.java -- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/authentication/BasicAuthProviderBuilder.java b/eagle-server/src/main/java/org/apache/eagle/server/authentication/BasicAuthProviderBuilder.java index e0bb4c1..855134b 100644 --- a/eagle-server/src/main/java/org/apache/eagle/server/authentication/BasicAuthProviderBuilder.java +++ b/eagle-server/src/main/java/org/apache/eagle/server/authentication/BasicAuthProviderBuilder.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.eagle.server.authentication; import com.google.common.cache.CacheBuilderSpec;
[35/50] incubator-eagle git commit: [EAGLE-625] remove site ignore
[EAGLE-625] remove site ignore Root `.gitignore` prevent the site folder which ignore the web page site folder. Author: zombieJCloses #516 from zombieJ/EAGLE-625. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/33b97206 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/33b97206 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/33b97206 Branch: refs/heads/master Commit: 33b9720656b8cd7ec5df883b79b20bccd5bddb3e Parents: 97edbbb Author: zombieJ Authored: Mon Oct 17 10:28:26 2016 +0800 Committer: zombieJ Committed: Mon Oct 17 10:28:26 2016 +0800 -- .gitignore | 1 - .../main/webapp/app/dev/partials/site/home.html | 36 2 files changed, 36 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/33b97206/.gitignore -- diff --git a/.gitignore b/.gitignore index 3a925f3..cfff821 100644 --- a/.gitignore +++ b/.gitignore @@ -83,5 +83,4 @@ logs/ *.orig **/*.pyc -site/ **/*.db \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/33b97206/eagle-server/src/main/webapp/app/dev/partials/site/home.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/site/home.html b/eagle-server/src/main/webapp/app/dev/partials/site/home.html new file mode 100644 index 000..217db56 --- /dev/null +++ b/eagle-server/src/main/webapp/app/dev/partials/site/home.html @@ -0,0 +1,36 @@ + + + + + + + + + + + No Widget + + No widget provided by Application + + + + + + +
[42/50] incubator-eagle git commit: [EAGLE-628] Fix jdbc metadata bugs
[EAGLE-628] Fix jdbc metadata bugs https://issues.apache.org/jira/browse/EAGLE-628 Author: wujinhuCloses #520 from wujinhu/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/47349674 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/47349674 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/47349674 Branch: refs/heads/master Commit: 473496747d42b9bfe72f32cb89c312eef1a360bf Parents: ecda7d5 Author: wujinhu Authored: Mon Oct 17 19:42:23 2016 +0800 Committer: wujinhu Committed: Mon Oct 17 19:42:23 2016 +0800 -- .../app/module/ApplicationGuiceModule.java | 8 +- .../eagle-metadata/eagle-metadata-jdbc/pom.xml | 5 + .../ApplicationEntityServiceJDBCImpl.java | 75 +++-- .../jdbc/service/SiteEntityServiceJDBCImpl.java | 16 +-- .../orm/ApplicationEntityToRelation.java| 5 + .../orm/RelationToApplicationEntity.java| 36 -- .../ApplicationEntityServiceJDBCImplTest.java | 40 --- .../store/jdbc/JDBCDataSourceProviderTest.java | 2 +- .../jdbc/JDBCMetadataQueryServiceTest.java | 2 +- .../store/jdbc/JDBCMetadataTestBase.java| 109 ++- .../store/jdbc/TestStaticApplication.java | 31 ++ ...tore.jdbc.TestStaticApplication$Provider.xml | 31 ++ ...org.apache.eagle.app.spi.ApplicationProvider | 16 +++ .../src/test/resources/init.sql | 7 +- .../service/JDBCSecurityMetadataDAO.java| 12 +- .../audit/JDBCSecurityMetadataDAOTest.java | 11 +- .../src/main/bin/createTables.sql | 40 +++ 17 files changed, 340 insertions(+), 106 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java index 67c7eec..d796faa 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/module/ApplicationGuiceModule.java @@ -16,9 +16,11 @@ */ package org.apache.eagle.app.module; +import com.typesafe.config.ConfigFactory; import org.apache.eagle.app.service.ApplicationManagementService; import org.apache.eagle.app.service.ApplicationProviderService; import org.apache.eagle.app.service.impl.ApplicationManagementServiceImpl; +import org.apache.eagle.app.service.impl.ApplicationProviderServiceImpl; import org.apache.eagle.app.service.impl.ApplicationStatusUpdateServiceImpl; import org.apache.eagle.metadata.service.ApplicationDescService; import com.google.inject.AbstractModule; @@ -27,12 +29,16 @@ import com.google.inject.util.Providers; import org.apache.eagle.metadata.service.ApplicationStatusUpdateService; public class ApplicationGuiceModule extends AbstractModule { -private ApplicationProviderService appProviderInst; +private final ApplicationProviderService appProviderInst; public ApplicationGuiceModule(ApplicationProviderService appProviderInst) { this.appProviderInst = appProviderInst; } +public ApplicationGuiceModule() { +this.appProviderInst = new ApplicationProviderServiceImpl(ConfigFactory.load()); +} + @Override protected void configure() { bind(ApplicationProviderService.class).toProvider(Providers.of(appProviderInst)); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml b/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml index 940d697..d15f30a 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/pom.xml @@ -46,5 +46,10 @@ com.h2database h2 + +org.apache.eagle +eagle-app-base +${project.version} + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/47349674/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/service/ApplicationEntityServiceJDBCImpl.java -- diff --git
[28/50] incubator-eagle git commit: [EAGLE-620]: AlertEngine: SpoutWrapper are sending duplicated message
[EAGLE-620]: AlertEngine: SpoutWrapper are sending duplicated message Author: ralphsu This closes #510 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2e715e3e Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2e715e3e Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2e715e3e Branch: refs/heads/master Commit: 2e715e3e94a6b98e0f3073d5d384878a0a845b72 Parents: c6ac2eb Author: Ralph, SuAuthored: Fri Oct 14 17:30:27 2016 +0800 Committer: Ralph, Su Committed: Fri Oct 14 17:32:25 2016 +0800 -- .../alert/coordinator/impl/MonitorMetadataGenerator.java| 2 +- .../alert/engine/spout/SpoutOutputCollectorWrapper.java | 9 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2e715e3e/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java index c5c992b..3f64f86 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java @@ -265,7 +265,7 @@ public class MonitorMetadataGenerator { } } if (targetSm == null) { -targetSm = new StreamRepartitionMetadata(datasourceName, schema.getStreamId()); +targetSm = new StreamRepartitionMetadata(topicName, schema.getStreamId()); dsStreamMeta.add(targetSm); } if (!targetSm.groupingStrategies.contains(gs)) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2e715e3e/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java index a8dcc0d..e205da4 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java @@ -18,6 +18,9 @@ */ package org.apache.eagle.alert.engine.spout; +import backtype.storm.spout.ISpoutOutputCollector; +import backtype.storm.spout.SpoutOutputCollector; +import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.eagle.alert.coordination.model.SpoutSpec; import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata; import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy; @@ -29,9 +32,6 @@ import org.apache.eagle.alert.engine.model.StreamEvent; import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer; import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider; import org.apache.eagle.alert.utils.StreamIdConversion; -import backtype.storm.spout.ISpoutOutputCollector; -import backtype.storm.spout.SpoutOutputCollector; -import org.apache.commons.lang3.builder.HashCodeBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,6 +121,9 @@ public class SpoutOutputCollectorWrapper extends SpoutOutputCollector implements phase 2: stream repartition */ for (StreamRepartitionMetadata md : streamRepartitionMetadataList) { +if (!event.getStreamId().equals(md.getStreamId())) { +continue; +} // one stream may have multiple group-by strategies, each strategy is for a specific group-by for (StreamRepartitionStrategy groupingStrategy : md.groupingStrategies) { int hash = 0;
[19/50] incubator-eagle git commit: EAGLE-594: Remove raw alert from kafka publisher
EAGLE-594: Remove raw alert from kafka publisher We are leveraging configured deduplicator to dedup the duplicated alerts before publish to kafka, email, slack, etc. However, sometimes we may still want to keep the raw alerts in kafka. Here we have defined rawAlertNamespaceLabel and rawAlertNamespaceValue custom fields to emit the raw alerts into kafka. However, these configured namespace concept is ebay/sherlock specific, we should remove it from eagle and use it ebay/sherlock extended kafka publisher. Author: Li, Garrett Reviewer: ralphsu This closes #478 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a3df0743 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a3df0743 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a3df0743 Branch: refs/heads/master Commit: a3df074379b3952c366e1d9d876d0223ca24ffed Parents: 1d0f9f5 Author: Xiancheng LiAuthored: Sun Oct 9 10:37:06 2016 +0800 Committer: Ralph, Su Committed: Thu Oct 13 16:51:23 2016 +0800 -- .../engine/publisher/PublishConstants.java | 3 - .../publisher/impl/AlertKafkaPublisher.java | 90 ++-- 2 files changed, 44 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3df0743/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java index 7408779..0ba2313 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java @@ -51,7 +51,4 @@ public class PublishConstants { public static final String ALERT_EMAIL_POLICY = "policyId"; public static final String ALERT_EMAIL_CREATOR = "creator"; -public static final String RAW_ALERT_NAMESPACE_LABEL = "rawAlertNamespaceLabel"; -public static final String RAW_ALERT_NAMESPACE_VALUE = "rawAlertNamespaceValue"; - } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3df0743/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java index 5464ded..5df34e3 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java @@ -18,7 +18,6 @@ package org.apache.eagle.alert.engine.publisher.impl; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,8 +44,6 @@ public class AlertKafkaPublisher extends AbstractPublishPlugin { private KafkaProducer producer; private String brokerList; private String topic; -private String namespaceLabel; -private String namespaceValue; @Override @SuppressWarnings("rawtypes") @@ -58,42 +55,64 @@ public class AlertKafkaPublisher extends AbstractPublishPlugin { brokerList = kafkaConfig.get(PublishConstants.BROKER_LIST).trim(); producer = KafkaProducerManager.INSTANCE.getProducer(brokerList, kafkaConfig); topic = kafkaConfig.get(PublishConstants.TOPIC).trim(); -namespaceLabel = kafkaConfig.getOrDefault(PublishConstants.RAW_ALERT_NAMESPACE_LABEL, "namespace"); -namespaceValue = kafkaConfig.getOrDefault(PublishConstants.RAW_ALERT_NAMESPACE_VALUE, "network"); } } -@SuppressWarnings( {"unchecked", "rawtypes"}) @Override public void onAlert(AlertStreamEvent event) throws Exception { if (producer == null) { LOG.warn("KafkaProducer is null due to the incorrect configurations"); return; } -List outputEvents = new ArrayList(); -int
[13/50] incubator-eagle git commit: EAGLE-609: Fix nullPointerException in DefaultDeduplicator
EAGLE-609: Fix nullPointerException in DefaultDeduplicator https://issues.apache.org/jira/browse/EAGLE-609 Author: Zhao, Qingwen <qingwz...@apache.org> Closes #492 from qingwen220/EAGLE-609. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/279bc01c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/279bc01c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/279bc01c Branch: refs/heads/master Commit: 279bc01cd6115281d44df7f81ac3b2be9f3b748f Parents: fc2407c Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Wed Oct 12 12:16:37 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Oct 12 12:16:37 2016 +0800 -- ...e.alert.app.AlertUnitTopologyAppProvider.xml | 6 ++ .../publisher/impl/DefaultDeduplicator.java | 21 +++- .../auditlog/HdfsAuditLogParserBolt.java| 8 3 files changed, 22 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/279bc01c/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index bf22123..0188210 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -25,6 +25,12 @@ +topology.name +topology.name +alertUnitTopology_1 +alert topology name + + topology.numOfTotalWorkers topology.numOfTotalWorkers 2 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/279bc01c/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java index a3969ab..ac99db3 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java @@ -116,19 +116,22 @@ public class DefaultDeduplicator implements AlertDeduplicator { continue; } String colName = streamDefinition.getColumns().get(i).getName(); +Object colValue = event.getData()[i]; -if (colName.equals(dedupStateField)) { -stateFiledValue = event.getData()[i].toString(); +if (colName.equals(dedupStateField) && colValue != null) { +stateFiledValue = colValue.toString(); } // make all of the field as unique key if no custom dedup field provided -if (customDedupFields == null || customDedupFields.size() <= 0) { -customFieldValues.put(colName, event.getData()[i].toString()); -} else { -for (String field : customDedupFields) { -if (colName.equals(field)) { -customFieldValues.put(field, event.getData()[i].toString()); -break; +if (colValue != null) { +if (customDedupFields == null || customDedupFields.size() <= 0) { +customFieldValues.put(colName, colValue.toString()); +} else { +for (String field : customDedupFields) { +if (colName.equals(field)) { +customFieldValues.put(field, colValue.toString()); +break; +} } } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/279bc01c/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/audi
[44/50] incubator-eagle git commit: [EAGLE-632] clean up configuration for MR history feeder
[EAGLE-632] clean up configuration for MR history feeder Author: wujinhuCloses #522 from wujinhu/EAGLE-632. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a781937e Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a781937e Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a781937e Branch: refs/heads/master Commit: a781937e978025b708ccc70d47ddaadc18668948 Parents: 3d6a29e Author: wujinhu Authored: Tue Oct 18 15:43:44 2016 +0800 Committer: wujinhu Committed: Tue Oct 18 15:43:44 2016 +0800 -- .../jpm/mr/history/MRHistoryJobApplication.java | 14 +- .../jpm/mr/history/MRHistoryJobConfig.java | 62 +--- .../crawler/DefaultJHFInputStreamCallback.java | 2 +- .../history/crawler/JHFCrawlerDriverImpl.java | 6 +- .../metrics/JobCountMetricsGenerator.java | 2 +- .../mr/history/parser/JHFEventReaderBase.java | 20 +- .../mr/history/parser/JHFMRVer1EventReader.java | 152 -- .../jpm/mr/history/parser/JHFMRVer1Parser.java | 283 --- .../parser/JHFMRVer1PerLineListener.java| 39 --- .../mr/history/parser/JHFMRVer2EventReader.java | 1 - .../jpm/mr/history/parser/JHFParserFactory.java | 44 +-- ...JobConfigurationCreationServiceListener.java | 2 +- .../JobEntityCreationEagleServiceListener.java | 4 +- .../parser/TaskAttemptCounterListener.java | 2 +- .../mr/history/parser/TaskFailureListener.java | 2 +- .../jpm/mr/history/storm/JobHistorySpout.java | 15 +- history.MRHistoryJobApplicationProvider.xml | 102 --- .../src/main/resources/application.conf | 44 +-- 18 files changed, 131 insertions(+), 665 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a781937e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java index beec938..de35678 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplication.java @@ -59,16 +59,12 @@ public class MRHistoryJobApplication extends StormApplication { JobHistoryContentFilter filter = builder.build(); //3. init topology TopologyBuilder topologyBuilder = new TopologyBuilder(); -String spoutName = "mrHistoryJobExecutor"; -int parallelism = jhfAppConf.getInt("envContextConfig.parallelismConfig." + spoutName); -int tasks = jhfAppConf.getInt("envContextConfig.tasks." + spoutName); -if (parallelism > tasks) { -parallelism = tasks; -} +String spoutName = "mrHistoryJobSpout"; +int tasks = jhfAppConf.getInt("stormConfig.mrHistoryJobSpoutTasks"); topologyBuilder.setSpout( -spoutName, -new JobHistorySpout(filter, config), -parallelism +spoutName, +new JobHistorySpout(filter, config), +tasks ).setNumTasks(tasks); return topologyBuilder.createTopology(); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a781937e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java index 37df5ad..561c3d0 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java @@ -49,18 +49,6 @@ public class MRHistoryJobConfig implements Serializable { private JobHistoryEndpointConfig jobHistoryEndpointConfig; -public ControlConfig getControlConfig() { -return controlConfig; -} - -private ControlConfig controlConfig; - -public JobExtractorConfig getJobExtractorConfig() { -return jobExtractorConfig; -} - -private JobExtractorConfig jobExtractorConfig; - public EagleServiceConfig getEagleServiceConfig() { return eagleServiceConfig; } @@ -79,25 +67,14 @@ public class MRHistoryJobConfig
[29/50] incubator-eagle git commit: [EAGLE-616] AlertEngine: Reduce the connection from alert-service to ZK
[EAGLE-616] AlertEngine: Reduce the connection from alert-service to ZK Author: Zeng, Bryant Reviewer: ralphsu This closes #501 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/98d1964f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/98d1964f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/98d1964f Branch: refs/heads/master Commit: 98d1964ff6bd19c706952fd4b09ad40dbc5a4870 Parents: 2e715e3 Author: mizengAuthored: Thu Oct 13 17:28:06 2016 +0800 Committer: Ralph, Su Committed: Fri Oct 14 17:38:16 2016 +0800 -- .../java/org/apache/eagle/alert/coordinator/Coordinator.java | 1 + .../eagle/alert/coordinator/trigger/CoordinatorTrigger.java| 6 -- 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/98d1964f/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java index dc3e3d6..e0bd5c3 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java @@ -185,6 +185,7 @@ public class Coordinator { for (String topo : state.getPublishSpecs().keySet()) { producer.send(MessageFormat.format(ZK_ALERT_CONFIG_PUBLISHER, topo), value); } + } public ScheduleState getState() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/98d1964f/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/CoordinatorTrigger.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/CoordinatorTrigger.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/CoordinatorTrigger.java index ce3c02b..e0e0612 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/CoordinatorTrigger.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/CoordinatorTrigger.java @@ -65,8 +65,10 @@ public class CoordinatorTrigger implements Runnable { ScheduleState state = scheduler.schedule(new ScheduleOption()); -ConfigBusProducer producer = new ConfigBusProducer(ZKConfigBuilder.getZKConfig(config)); -Coordinator.postSchedule(client, state, producer); +// use try catch to use AutoCloseable interface to close producer automatically +try (ConfigBusProducer producer = new ConfigBusProducer(ZKConfigBuilder.getZKConfig(config))) { +Coordinator.postSchedule(client, state, producer); +} watch.stop(); LOG.info("CoordinatorTrigger ended, used time {} sm.", watch.elapsed(TimeUnit.MILLISECONDS));
[22/50] incubator-eagle git commit: [EAGLE-617] Scrollbar is missing when install Map Reduce History Job Monitoring
[EAGLE-617] Scrollbar is missing when install Map Reduce History Job Monitoring add check logic in uiSrv. When dialog close, will check current existed dialog for body 'modal-open' Author: zombieJCloses #503 from zombieJ/EAGLE-617. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0bca234f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0bca234f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0bca234f Branch: refs/heads/master Commit: 0bca234ff8d6b747a62853ff1f3028fc9ab1e4f9 Parents: 5e57c7b Author: zombieJ Authored: Fri Oct 14 11:28:11 2016 +0800 Committer: zombieJ Committed: Fri Oct 14 11:28:11 2016 +0800 -- .../src/main/webapp/app/dev/public/js/services/uiSrv.js | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0bca234f/eagle-server/src/main/webapp/app/dev/public/js/services/uiSrv.js -- diff --git a/eagle-server/src/main/webapp/app/dev/public/js/services/uiSrv.js b/eagle-server/src/main/webapp/app/dev/public/js/services/uiSrv.js index e0bb77d..6ce77b8 100644 --- a/eagle-server/src/main/webapp/app/dev/public/js/services/uiSrv.js +++ b/eagle-server/src/main/webapp/app/dev/public/js/services/uiSrv.js @@ -94,6 +94,10 @@ entity: _entity }); $mdl.remove(); + + if ($(".modal-backdrop").length) { + $("body").addClass("modal-open"); + } }); // Function @@ -225,6 +229,10 @@ name: name }); $mdl.remove(); + + if ($(".modal-backdrop").length) { + $("body").addClass("modal-open"); + } }); // Function
[46/50] incubator-eagle git commit: [EAGLE-633] MR history feeder read some configurations from eagle server
[EAGLE-633] MR history feeder read some configurations from eagle server Author: wujinhuCloses #525 from wujinhu/EAGLE-633. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2479d327 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2479d327 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2479d327 Branch: refs/heads/master Commit: 2479d32791dd1a424af7513279ba28543baf6af4 Parents: 310935d Author: wujinhu Authored: Tue Oct 18 17:20:37 2016 +0800 Committer: wujinhu Committed: Tue Oct 18 17:20:37 2016 +0800 -- .../jpm/mr/history/MRHistoryJobConfig.java | 20 +++--- history.MRHistoryJobApplicationProvider.xml | 64 +--- .../src/main/resources/application.conf | 4 +- eagle-server-assembly/src/main/conf/eagle.conf | 10 +++ .../src/main/resources/application.conf | 10 +++ 5 files changed, 33 insertions(+), 75 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2479d327/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java index 561c3d0..566f7d0 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java @@ -119,11 +119,11 @@ public class MRHistoryJobConfig implements Serializable { this.config = config; //parse eagle zk -this.zkStateConfig.zkQuorum = config.getString("zkStateConfig.zkQuorum"); -this.zkStateConfig.zkSessionTimeoutMs = config.getInt("zkStateConfig.zkSessionTimeoutMs"); -this.zkStateConfig.zkRetryTimes = config.getInt("zkStateConfig.zkRetryTimes"); -this.zkStateConfig.zkRetryInterval = config.getInt("zkStateConfig.zkRetryInterval"); -this.zkStateConfig.zkRoot = config.getString("zkStateConfig.zkRoot"); +this.zkStateConfig.zkQuorum = config.getString("zookeeper.zkQuorum"); +this.zkStateConfig.zkSessionTimeoutMs = config.getInt("zookeeper.zkSessionTimeoutMs"); +this.zkStateConfig.zkRetryTimes = config.getInt("zookeeper.zkRetryTimes"); +this.zkStateConfig.zkRetryInterval = config.getInt("zookeeper.zkRetryInterval"); +this.zkStateConfig.zkRoot = config.getString("zookeeper.zkRoot"); //parse job history endpoint this.jobHistoryEndpointConfig.site = config.getString("siteId"); @@ -135,12 +135,12 @@ public class MRHistoryJobConfig implements Serializable { this.jobHistoryEndpointConfig.timeZone = config.getString("endpointConfig.timeZone"); // parse eagle service endpoint -this.eagleServiceConfig.eagleServiceHost = config.getString("eagleService.host"); -String port = config.getString("eagleService.port"); +this.eagleServiceConfig.eagleServiceHost = config.getString("service.host"); +String port = config.getString("service.port"); this.eagleServiceConfig.eagleServicePort = (port == null ? 8080 : Integer.parseInt(port)); -this.eagleServiceConfig.username = config.getString("eagleService.username"); -this.eagleServiceConfig.password = config.getString("eagleService.password"); -this.eagleServiceConfig.readTimeoutSeconds = config.getInt("eagleService.readTimeOutSeconds"); +this.eagleServiceConfig.username = config.getString("service.username"); +this.eagleServiceConfig.password = config.getString("service.password"); +this.eagleServiceConfig.readTimeoutSeconds = config.getInt("service.readTimeOutSeconds"); LOG.info("Successfully initialized MRHistoryJobConfig"); LOG.info("zkStateConfig.zkQuorum: " + this.zkStateConfig.zkQuorum); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2479d327/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml -- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml index 9835411..42f3167 100644 ---
[14/50] incubator-eagle git commit: [EAGLE-613] Support modify ApplicationEntity through POST /rest/apps/{UUID}/
[EAGLE-613] Support modify ApplicationEntity through POST /rest/apps/{UUID}/ Support modify ApplicationEntity through ~~~ POST /rest/apps/{UUID}/ ~~~ * REQUEST BODY* (UpdateOperation) ~~~ { "configuration": { "spout.stormKafkaUseSameZkQuorumWithKafkaBroker": "true", "metadataService.port": "9090", "spout.stormKafkaEagleConsumer": "eagle_consumer", "topology.numOfSpoutTasks": "1", "jarPath": "incubator-eagle/eagle-core/eagle-alert-parent/eagle-alert-app/target/classes/org/apache/eagle/alert/app/AlertUnitTopologyApp.class", "topology.numOfRouterBolts": "4", "topology.numOfAlertBolts": "10", "zkConfig.zkRoot": "/alert", "zkConfig.zkQuorum": "localhost:2181", "mode": "LOCAL", "metadataService.context": "/rest", "metadataService.host": "localhost", "topology.messageTimeoutSecs": "3600", "spout.kafkaBrokerZkQuorum": "localhost:2181", "appId": "ALERTUNITTOPOLOGYAPP_SANDBOX", "topology.numOfTotalWorkers": "2", "siteId": "sandbox", "topology.numOfPublishTasks": "1", "spout.stormKafkaTransactionZkPath": "/consumers", "spout.kafkaBrokerZkBasePath": "/brokers" }, "mode": "CLUSTER", "jarPath": "incubator-eagle/eagle-core/eagle-alert-parent/eagle-alert-app/target/classes/org/apache/eagle/alert/app/AlertUnitTopologyApp.class" } ~~~ Author: Hao Chen <h...@apache.org> Closes #495 from haoch/EAGLE-613. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6823410f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6823410f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6823410f Branch: refs/heads/master Commit: 6823410fc85fe13d9295125d9093754d38e4f0ac Parents: 279bc01 Author: Hao Chen <h...@apache.org> Authored: Wed Oct 12 18:01:08 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Oct 12 18:01:08 2016 +0800 -- .../eagle/app/resource/ApplicationResource.java | 14 .../app/service/ApplicationOperations.java | 35 .../eagle/metadata/model/ApplicationEntity.java | 11 ++ .../service/ApplicationEntityService.java | 1 + .../ApplicationEntityServiceMemoryImpl.java | 9 + .../ApplicationEntityServiceJDBCImpl.java | 9 - 6 files changed, 78 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6823410f/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/resource/ApplicationResource.java -- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/resource/ApplicationResource.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/resource/ApplicationResource.java index 717f7c0..b493b2b 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/resource/ApplicationResource.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/resource/ApplicationResource.java @@ -91,6 +91,20 @@ public class ApplicationResource { } @POST +@Path("/{appUuid}") +@Produces(MediaType.APPLICATION_JSON) +public RESTResponse updateApplicationEntity(@PathParam("appUuid") String appUuid, ApplicationOperations.UpdateOperation updateOperation) { +return RESTResponse.async(() -> { +ApplicationEntity applicationEntity = new ApplicationEntity(); +applicationEntity.setUuid(appUuid); +applicationEntity.setJarPath(updateOperation.getJarPath()); +applicationEntity.setMode(updateOperation.getMode()); + applicationEntity.setConfiguration(updateOperation.getConfiguration()); +return entityService.update(applicationEntity); +}).get(); +} + +@POST @Path("/status") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6823410f/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java -- diff --git a/eagle-core
[24/50] incubator-eagle git commit: [EAGLE-582] add simple Basic Authentication registration to server appl…
[EAGLE-582] add simple Basic Authentication registration to server appl⦠This is one of the sub-tasks of ticket EAGLE-433, to successfully register a simple Basic Authentication authenticator to the server application. With this check-in, when the server is up, any access to rest api method that has a Auth annotation on it's User field will require correct credentials. Author: anyway1021Closes #468 from anyway1021/EAGLE-582. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/649adb1a Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/649adb1a Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/649adb1a Branch: refs/heads/master Commit: 649adb1ae400e026a8e932d53bd813f03fecb2d1 Parents: 7bd5d1d Author: anyway1021 Authored: Fri Oct 14 12:13:23 2016 +0800 Committer: anyway1021 Committed: Fri Oct 14 12:13:23 2016 +0800 -- .../eagle/common/authentication/User.java | 45 ++ eagle-server/pom.xml| 23 +++ .../apache/eagle/server/ServerApplication.java | 22 +-- .../org/apache/eagle/server/ServerConfig.java | 14 ++ .../authentication/AuthenticationMode.java | 51 +++ .../AuthenticationModeIdentifier.java | 105 ++ .../authentication/AuthenticationRegister.java | 41 ++ .../SwitchableBasicAuthProvider.java| 50 +++ .../AbstractSwitchableAuthenticator.java| 48 +++ .../authenticator/LdapBasicAuthenticator.java | 40 ++ .../authenticator/SimpleBasicAuthenticator.java | 43 ++ .../config/AuthenticationSettings.java | 119 .../authentication/config/LdapSettings.java | 142 +++ .../authentication/config/SimpleSettings.java | 46 ++ .../src/main/resources/application.conf | 2 +- .../src/main/resources/configuration.yml| 48 ++- .../TestBasicAuthenticationResource.java| 39 + .../src/test/resources/configuration.yml| 45 ++ pom.xml | 13 +- 19 files changed, 924 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/649adb1a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/authentication/User.java -- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/authentication/User.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/authentication/User.java new file mode 100644 index 000..85388cd --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/authentication/User.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.common.authentication; + +import java.io.Serializable; +import java.security.Principal; +import java.util.Set; + +public class User implements Principal, Serializable { +private String username = "Unauthenticated"; +private Set roles = null; + +public User() {} + +public User(String username) { +this.username = username; +} + +public User(String username, Set roles) { +this.username = username; +this.roles = roles; +} + +public Set getRoles() { +return roles; +} + +public String getName() { +return username; +} +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/649adb1a/eagle-server/pom.xml -- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index 21bda30..7fe4397 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -133,6 +133,14 @@ org.wso2.orbit.com.lmax disruptor + +org.slf4j +slf4j-log4j12 + + +
[36/50] incubator-eagle git commit: [EAGLE-621] set authentication disabled by default when no 'auth' is configured
[EAGLE-621] set authentication disabled by default when no 'auth' is configured Authentication settings are configured in configuration.yml file, starting with "auth" syntax. Now make the system see authentication as disabled when no auth related syntax is set. Also, do some refactoring to make the code clearer. Author: anyway1021Closes #514 from anyway1021/EAGLE-621. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b103e201 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b103e201 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b103e201 Branch: refs/heads/master Commit: b103e201ebe7b1500ba3d0297bb7aa88e7aa811a Parents: 33b9720 Author: anyway1021 Authored: Mon Oct 17 10:44:36 2016 +0800 Committer: anyway1021 Committed: Mon Oct 17 10:44:36 2016 +0800 -- .../apache/eagle/server/ServerApplication.java | 8 +- .../authentication/AuthenticationMode.java | 51 - .../AuthenticationModeIdentifier.java | 105 --- .../authentication/AuthenticationRegister.java | 41 .../BasicAuthProviderBuilder.java | 66 .../SwitchableBasicAuthProvider.java| 50 - .../AbstractSwitchableAuthenticator.java| 48 - .../authenticator/LdapBasicAuthenticator.java | 11 +- .../authenticator/SimpleBasicAuthenticator.java | 12 +-- 9 files changed, 82 insertions(+), 310 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b103e201/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java -- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java index 2ae95a6..df1d7e7 100644 --- a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java +++ b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.eagle.server; + import com.google.inject.Injector; import com.hubspot.dropwizard.guice.GuiceBundle; import com.sun.jersey.api.core.PackagesResourceConfig; @@ -27,11 +28,10 @@ import io.swagger.jaxrs.config.BeanConfig; import io.swagger.jaxrs.listing.ApiListingResource; import org.apache.eagle.alert.coordinator.CoordinatorListener; import org.apache.eagle.alert.resource.SimpleCORSFiler; -import org.apache.eagle.common.authentication.User; import org.apache.eagle.log.base.taggedlog.EntityJsonModule; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; import org.apache.eagle.metadata.service.ApplicationStatusUpdateService; -import org.apache.eagle.server.authentication.AuthenticationRegister; +import org.apache.eagle.server.authentication.BasicAuthProviderBuilder; import org.apache.eagle.server.managedtask.ApplicationTask; import org.apache.eagle.server.module.GuiceBundleLoader; @@ -80,8 +80,8 @@ class ServerApplication extends Application { environment.servlets().addFilter(SimpleCORSFiler.class.getName(), new SimpleCORSFiler()) .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); -// add authentication filters -new AuthenticationRegister<>(configuration, environment, User.class).register(); +// register authentication provider +environment.jersey().register(new BasicAuthProviderBuilder(configuration.getAuth(), environment).build()); // context listener environment.servlets().addServletListeners(new CoordinatorListener()); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b103e201/eagle-server/src/main/java/org/apache/eagle/server/authentication/AuthenticationMode.java -- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/authentication/AuthenticationMode.java b/eagle-server/src/main/java/org/apache/eagle/server/authentication/AuthenticationMode.java deleted file mode 100644 index 8a7208f..000 --- a/eagle-server/src/main/java/org/apache/eagle/server/authentication/AuthenticationMode.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *
[30/50] incubator-eagle git commit: [EAGLE-618] migration eagle-jpm-aggregation to application framework-bug fix
[EAGLE-618] migration eagle-jpm-aggregation to application framework-bug fix Author: wujinhuCloses #509 from wujinhu/EAGLE-618. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8ff8ecf1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8ff8ecf1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8ff8ecf1 Branch: refs/heads/master Commit: 8ff8ecf1b8d9ace5c87ad6acc30235b4a43d98bf Parents: 98d1964 Author: wujinhu Authored: Fri Oct 14 17:46:33 2016 +0800 Committer: wujinhu Committed: Fri Oct 14 17:46:33 2016 +0800 -- .../org/apache/eagle/jpm/aggregation/AggregationApplication.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8ff8ecf1/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java -- diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java index b040d3e..214c688 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java @@ -35,13 +35,13 @@ public class AggregationApplication extends StormApplication { List metricNames = new ArrayList<>(); String[] metricNamesArr = config.getString("aggregate.counters.metrics").split(","); for (int i = 0; i < metricNamesArr.length; i++) { -metricNames.add(metricNamesArr[i]); +metricNames.add(metricNamesArr[i].trim()); } List groupByColumns = new ArrayList<>(); String[] groupByColumnsArr = config.getString("aggregate.counters.groupBys").split(";"); for (int i = 0; i < groupByColumnsArr.length; i++) { -groupByColumns.add(groupByColumnsArr[i]); +groupByColumns.add(groupByColumnsArr[i].trim()); } Map > metrics = new HashMap<>();
[18/50] incubator-eagle git commit: [EAGLE-604] Support view/editing application configuration after installed
[EAGLE-604] Support view/editing application configuration after installed Support view/editing application configuration after installed Author: zombieJCloses #500 from zombieJ/EAGLE-604. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/1d0f9f5b Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/1d0f9f5b Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/1d0f9f5b Branch: refs/heads/master Commit: 1d0f9f5b03acefbfb9af97a83ca19ad3d02d638a Parents: 4dfb528 Author: zombieJ Authored: Thu Oct 13 16:13:08 2016 +0800 Committer: zombieJ Committed: Thu Oct 13 16:13:08 2016 +0800 -- .../webapp/app/dev/partials/alert/main.html | 63 .../app/dev/partials/alert/policyDetail.html| 40 ++--- .../app/dev/partials/alert/streamList.html | 47 ++ .../app/dev/partials/integration/main.html | 1 - .../app/dev/partials/integration/site.html | 145 +-- .../dev/partials/integration/streamList.html| 52 --- .../src/main/webapp/app/dev/public/js/app.js| 14 +- .../src/main/webapp/app/dev/public/js/common.js | 18 ++- .../webapp/app/dev/public/js/ctrls/alertCtrl.js | 19 +++ .../app/dev/public/js/ctrls/alertEditCtrl.js| 5 +- .../app/dev/public/js/ctrls/integrationCtrl.js | 135 - .../src/main/webapp/app/dev/public/js/index.js | 9 +- .../dev/public/js/services/applicationSrv.js| 4 + .../app/dev/public/js/services/pageSrv.js | 6 +- 14 files changed, 386 insertions(+), 172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1d0f9f5b/eagle-server/src/main/webapp/app/dev/partials/alert/main.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/alert/main.html b/eagle-server/src/main/webapp/app/dev/partials/alert/main.html index 2e062a8..2e14db9 100644 --- a/eagle-server/src/main/webapp/app/dev/partials/alert/main.html +++ b/eagle-server/src/main/webapp/app/dev/partials/alert/main.html @@ -1,29 +1,34 @@ - - - - - Explore Triggered Alerts - Manage Policies - Define Alert Policy - - - - - - + + + + + Alerts + Policies + Streams + + Define Alert Policy + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1d0f9f5b/eagle-server/src/main/webapp/app/dev/partials/alert/policyDetail.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/alert/policyDetail.html b/eagle-server/src/main/webapp/app/dev/partials/alert/policyDetail.html index b04a8a8..7ec5f39 100644 --- a/eagle-server/src/main/webapp/app/dev/partials/alert/policyDetail.html +++ b/eagle-server/src/main/webapp/app/dev/partials/alert/policyDetail.html @@ -18,7 +18,8 @@ - Policy + Basic + Definition Publisher @@ -30,7 +31,7 @@ - + @@ -61,26 +62,29 @@ - - Definition - {{policy.definition.value}} - - - Partition - - - - [{{partition.type}}] - {{partition.streamId}}: - {{partition.columns.join(", ")}} - - - - + + Definition + {{policy.definition.value}} + + Partition + +
[23/50] incubator-eagle git commit: [EAGLE-618] migration eagle-jpm-aggregation to application framework
[EAGLE-618] migration eagle-jpm-aggregation to application framework Author: wujinhuCloses #504 from wujinhu/EAGLE-618. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7bd5d1d2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7bd5d1d2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7bd5d1d2 Branch: refs/heads/master Commit: 7bd5d1d2adb909eef7f59424f758fc0818fb683b Parents: 0bca234 Author: wujinhu Authored: Fri Oct 14 11:56:03 2016 +0800 Committer: wujinhu Committed: Fri Oct 14 11:56:03 2016 +0800 -- .../jpm/aggregation/AggregationApplication.java | 18 ++- .../jpm/aggregation/AggregationConfig.java | 10 -- ...gregation.AggregationApplicationProvider.xml | 115 +++ .../src/main/resources/application.conf | 23 +--- eagle-topology-assembly/pom.xml | 5 + ...org.apache.eagle.app.spi.ApplicationProvider | 3 +- 6 files changed, 137 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7bd5d1d2/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java -- diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java index 3c40f58..0577070 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java @@ -31,22 +31,30 @@ import java.util.*; public class AggregationApplication extends StormApplication { @Override public StormTopology execute(Config config, StormEnvironment environment) { -AggregationConfig aggregationConfig = AggregationConfig.getInstance(config); - //TODO -List metricNames = config.getStringList("aggregate.counters.metrics"); -List groupByColumns = config.getStringList("aggregate.counters.groupByColumns"); +List metricNames = new ArrayList<>(); +String[] metricNamesArr = config.getString("aggregate.counters.metrics").split(","); +for (int i = 0; i < metricNamesArr.length; i++) { +metricNames.add(metricNamesArr[i]); +} +List groupByColumns = new ArrayList<>(); +String[] groupByColumnsArr = config.getString("aggregate.counters.groupBys").split(","); +for (int i = 0; i < groupByColumnsArr.length; i++) { +groupByColumns.add(groupByColumnsArr[i]); +} + Map > metrics = new HashMap<>(); for (String metric : metricNames) { metrics.put(metric, new ArrayList<>()); for (String cols : groupByColumns) { -metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", "").split(","))); +metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", "").split("&"))); } } TopologyBuilder topologyBuilder = new TopologyBuilder(); String spoutName = "mrHistoryAggregationSpout"; String boltName = "mrHistoryAggregationBolt"; +AggregationConfig aggregationConfig = AggregationConfig.getInstance(config); int parallelism = aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + spoutName); int tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." + spoutName); if (parallelism > tasks) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7bd5d1d2/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java -- diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java index c50de92..b527ddb 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java @@ -26,12 +26,6 @@ import java.io.Serializable; public class AggregationConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(AggregationConfig.class); -public String getEnv() { -return env; -} - -private String env; - public ZKStateConfig
[47/50] incubator-eagle git commit: [MINOR] Fix windows EOL
[MINOR] Fix windows EOL Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/76fc4410 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/76fc4410 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/76fc4410 Branch: refs/heads/master Commit: 76fc4410bbfcddc4757eeb6ebabaa8baf658036a Parents: 2479d32 Author: Hao Chen <h...@apache.org> Authored: Tue Oct 18 17:59:55 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Oct 18 17:59:55 2016 +0800 -- .../eagle/service/hbase/EmbeddedHbaseTest.java | 50 ++--- .../store/jdbc/JDBCDataSourceProviderTest.java | 148 ++--- .../jdbc/JDBCMetadataQueryServiceTest.java | 72 +++ eagle-dev/ci-log4j.properties | 42 ++-- .../jpm/mr/history/MRHistoryJobApplication.java | 142 ++--- history.MRHistoryJobApplicationProvider.xml | 206 +-- .../audit/JDBCSecurityMetadataDAOTest.java | 138 ++--- 7 files changed, 399 insertions(+), 399 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/76fc4410/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java -- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java index 0330eee..e65f062 100644 --- a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java +++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java @@ -1,26 +1,26 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.hbase; - -import org.junit.Test; - -public class EmbeddedHbaseTest extends TestHBaseBase { -@Test -public void testHBaseCreateTable() { -// hbase.createTable("test_hbase_table","f"); -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.service.hbase; + +import org.junit.Test; + +public class EmbeddedHbaseTest extends TestHBaseBase { +@Test +public void testHBaseCreateTable() { +// hbase.createTable("test_hbase_table","f"); +} } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/76fc4410/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java -- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java index 545ade4..1af1839 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java +++ b/eagle-core/eagle-metadata/eagle-meta
[26/50] incubator-eagle git commit: [MINOR] update AlertUnitTopologyAppProvider.xml
[MINOR] update AlertUnitTopologyAppProvider.xml Author: Zhao, QingwenCloses #507 from qingwen220/quickFix. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/e6f9d1bb Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/e6f9d1bb Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/e6f9d1bb Branch: refs/heads/master Commit: e6f9d1bbd6894c31b60bec4d23f42da07e6fe713 Parents: f3204bc Author: Zhao, Qingwen Authored: Fri Oct 14 15:36:28 2016 +0800 Committer: Zhao, Qingwen Committed: Fri Oct 14 15:36:28 2016 +0800 -- ...g.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml | 8 1 file changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/e6f9d1bb/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index bf22123..8e386be 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -25,10 +25,10 @@ -topology.numOfTotalWorkers -topology.numOfTotalWorkers -2 -number of total storm workers +workers +workers +1 +number of topology workers topology.numOfSpoutTasks
[25/50] incubator-eagle git commit: [EAGLE-612] Use LOCAL when app provider is from class file and use Cluster when from jar file
[EAGLE-612] Use LOCAL when app provider is from class file and use Cluster when from jar file When jarFile is jar file and use CLUSTER by default. (end with *.jar) When jarFile is class and use LOCAL by default. (end with *.class) Author: zombieJCloses #506 from zombieJ/EAGLE-612. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f3204bca Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f3204bca Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f3204bca Branch: refs/heads/master Commit: f3204bcafc345edc9b600715776af77d0d1f4496 Parents: 649adb1 Author: zombieJ Authored: Fri Oct 14 13:51:10 2016 +0800 Committer: zombieJ Committed: Fri Oct 14 13:51:10 2016 +0800 -- eagle-dev/merge-eagle-pr.py | 2 ++ .../main/webapp/app/dev/partials/integration/site.html| 2 +- .../webapp/app/dev/public/js/ctrls/integrationCtrl.js | 10 ++ 3 files changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f3204bca/eagle-dev/merge-eagle-pr.py -- diff --git a/eagle-dev/merge-eagle-pr.py b/eagle-dev/merge-eagle-pr.py index 01661d5..9243265 100755 --- a/eagle-dev/merge-eagle-pr.py +++ b/eagle-dev/merge-eagle-pr.py @@ -105,9 +105,11 @@ def clean_up(): print "Restoring head pointer to %s" % original_head run_cmd("git checkout %s" % original_head) +ansi_escape = re.compile(r'\x1b[^m]*m') branches = run_cmd("git branch").replace(" ", "").split("\n") for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches): +branch = ansi_escape.sub('', branch) print "Deleting local branch %s" % branch run_cmd("git branch -D %s" % branch) http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f3204bca/eagle-server/src/main/webapp/app/dev/partials/integration/site.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/integration/site.html b/eagle-server/src/main/webapp/app/dev/partials/integration/site.html index e320c48..98e7d97 100644 --- a/eagle-server/src/main/webapp/app/dev/partials/integration/site.html +++ b/eagle-server/src/main/webapp/app/dev/partials/integration/site.html @@ -237,7 +237,7 @@ jar Path - + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f3204bca/eagle-server/src/main/webapp/app/dev/public/js/ctrls/integrationCtrl.js -- diff --git a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/integrationCtrl.js b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/integrationCtrl.js index ca6c92e..e53a11e 100644 --- a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/integrationCtrl.js +++ b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/integrationCtrl.js @@ -188,6 +188,15 @@ return pass; }; + $scope.checkJarPath = function () { + var jarPath = ($scope.tmpApp || {}).jarPath; + if(/\.jar$/.test(jarPath)) { + $scope.tmpApp.mode = "CLUSTER"; + } else if(/\.class/.test(jarPath)) { + $scope.tmpApp.mode = "LOCAL"; + } + }; + $scope.installAppConfirm = function () { $scope.installLock = true; @@ -236,6 +245,7 @@ siteId: $scope.site.siteId, appType: application.type }); + $scope.checkJarPath(); } var fields = $scope.tmpAppConfigFields = common.getValueByPath(application, "configuration.properties", []).concat();
[49/50] incubator-eagle git commit: [UI] Improve integeration page
[UI] Improve integeration page Be sure to do all of the following to help us incorporate your contribution quickly and easily: - [x] Make sure the PR title is formatted like: `[EAGLE-] Description of pull request` - [x] Make sure tests pass via `mvn clean verify`. (Even better, enable Travis-CI on your fork and ensure the whole test matrix passes). - [x] Replace `` in the title with the actual Jira issue number, if there is one. - [x] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt). --- Author: Hao Chen <h...@apache.org> Closes #524 from haoch/ImproveIntegerationPage. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/adc2ba90 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/adc2ba90 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/adc2ba90 Branch: refs/heads/master Commit: adc2ba900a2d29352e96489ad48addde17bedccf Parents: c3bde01 Author: Hao Chen <h...@apache.org> Authored: Tue Oct 18 19:31:48 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Oct 18 19:31:48 2016 +0800 -- ...e.alert.app.AlertUnitTopologyAppProvider.xml | 83 +++- eagle-server/pom.xml| 10 +++ .../app/dev/partials/integration/site.html | 50 ++-- .../app/dev/public/js/ctrls/integrationCtrl.js | 8 +- eagle-server/ui-build.sh| 0 5 files changed, 87 insertions(+), 64 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/adc2ba90/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index b3e777c..6ef96c7 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -20,110 +20,123 @@ AlertUnitTopologyApp Alert Unit Topology Application 0.5.0-incubating -org.apache.eagle.alert.app.AlertUnitTopologyApp -/apps/alert workers -workers +Worker Number 1 -number of topology workers +Storm parallelism worker number +false topology.numOfSpoutTasks -topology.numOfSpoutTasks +Source Tasks Number 1 -number of spout tasks +Number of spout tasks +false topology.numOfRouterBolts -topology.numOfRouterBolts +Route Tasks Number 4 -number of router tasks +Number of router tasks +false topology.numOfAlertBolts -topology.numOfAlertBolts +Evaluator Tasks Number 10 -number of alert tasks +Number of alert tasks +false topology.numOfPublishTasks -topology.numOfPublishTasks +Publisher Tasks Number 1 -number of publish tasks +Number of publish tasks +false topology.messageTimeoutSecs -topology.messageTimeoutSecs +Message Timeout Seconds 3600 -number of tuple timeout in seconds +Number of tuple timeout in seconds +false spout.kafkaBrokerZkQuorum -spout.kafkaBrokerZkQuorum +Kafka Zookeeper Quorum localhost:2181 -zookeeper quorum for spout to consume data +Zookeeper quorum of kafka broker for spout to consume data +true spout.kafkaBrokerZkBasePath -spout.kafkaBrokerZkBasePath +Kafka Zookeeper Root /brokers -zk znode path for kafka brokers +Zookeeper znode path for kafka brokers +false spout.stormKafkaUseSameZkQuo
[17/50] incubator-eagle git commit: [EAGLE-614]: NPE in DynamicPolicyLoader
[EAGLE-614]: NPE in DynamicPolicyLoader Author: Zeng, Bryant Reviewer: ralphsu This closes #498 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4dfb528c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4dfb528c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4dfb528c Branch: refs/heads/master Commit: 4dfb528c4bf2d8935bac78b10a82b6bc15f0be59 Parents: 74c3cbb Author: mizengAuthored: Thu Oct 13 09:41:15 2016 +0800 Committer: Ralph, Su Committed: Thu Oct 13 15:49:03 2016 +0800 -- .../apache/eagle/alert/engine/coordinator/PolicyDefinition.java | 5 - .../eagle/alert/coordinator/trigger/DynamicPolicyLoader.java| 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4dfb528c/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java index 680b21a..6df682a 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java @@ -139,15 +139,18 @@ public class PolicyDefinition implements Serializable { if (that == this) { return true; } + if (!(that instanceof PolicyDefinition)) { return false; } + PolicyDefinition another = (PolicyDefinition) that; + if (Objects.equals(another.name, this.name) && Objects.equals(another.description, this.description) && CollectionUtils.isEqualCollection(another.inputStreams, this.inputStreams) && CollectionUtils.isEqualCollection(another.outputStreams, this.outputStreams) -&& another.definition.equals(this.definition) +&& (another.definition != null && another.definition.equals(this.definition)) && Objects.equals(this.definition, another.definition) && CollectionUtils.isEqualCollection(another.partitionSpec, this.partitionSpec) // && another.parallelismHint == this.parallelismHint http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4dfb528c/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java index a60c959..07ae966 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java @@ -66,7 +66,8 @@ public class DynamicPolicyLoader implements Runnable { List reallyModifiedPolicies = new ArrayList<>(); for (String updatedPolicy : potentiallyModifiedPolicies) { -if (!currPolicies.get(updatedPolicy).equals(cachedPolicies.get(updatedPolicy))) { +if (currPolicies.get(updatedPolicy) != null +&& !currPolicies.get(updatedPolicy).equals(cachedPolicies.get(updatedPolicy))) { reallyModifiedPolicies.add(updatedPolicy); } }
[43/50] incubator-eagle git commit: [MINOR] Set task number of each bolt in HdfsAuditLog Application
[MINOR] Set task number of each bolt in HdfsAuditLog Application If not set, the task number of all bolts are 8 Author: Zhao, QingwenCloses #521 from qingwen220/quickFix. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3d6a29ec Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3d6a29ec Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3d6a29ec Branch: refs/heads/master Commit: 3d6a29ec2e67a83742158a08d378cfc5cad59814 Parents: 4734967 Author: Zhao, Qingwen Authored: Mon Oct 17 22:45:19 2016 +0800 Committer: Zhao, Qingwen Committed: Mon Oct 17 22:45:19 2016 +0800 -- .../auditlog/AbstractHdfsAuditLogApplication.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3d6a29ec/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java -- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java index b985daf..b9f480b 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java @@ -59,11 +59,11 @@ public abstract class AbstractHdfsAuditLogApplication extends StormApplication { int numOfIPZoneJoinTasks = config.getInt(IPZONE_JOIN_TASK_NUM); int numOfSinkTasks = config.getInt(SINK_TASK_NUM); -builder.setSpout("ingest", spout, numOfSpoutTasks); +builder.setSpout("ingest", spout, numOfSpoutTasks).setNumTasks(numOfSpoutTasks); BaseRichBolt parserBolt = getParserBolt(); -BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks); +BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks).setNumTasks(numOfParserTasks); Boolean useDefaultPartition = !config.hasPath("eagleProps.useDefaultPartition") || config.getBoolean("eagleProps.useDefaultPartition"); if(useDefaultPartition){ @@ -73,15 +73,15 @@ public abstract class AbstractHdfsAuditLogApplication extends StormApplication { } HdfsSensitivityDataEnrichBolt sensitivityDataJoinBolt = new HdfsSensitivityDataEnrichBolt(config); -BoltDeclarer sensitivityDataJoinBoltDeclarer = builder.setBolt("sensitivityJoin", sensitivityDataJoinBolt, numOfSensitivityJoinTasks); +BoltDeclarer sensitivityDataJoinBoltDeclarer = builder.setBolt("sensitivityJoin", sensitivityDataJoinBolt, numOfSensitivityJoinTasks).setNumTasks(numOfSensitivityJoinTasks); sensitivityDataJoinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1")); IPZoneDataEnrichBolt ipZoneDataJoinBolt = new IPZoneDataEnrichBolt(config); -BoltDeclarer ipZoneDataJoinBoltDeclarer = builder.setBolt("ipZoneJoin", ipZoneDataJoinBolt, numOfIPZoneJoinTasks); +BoltDeclarer ipZoneDataJoinBoltDeclarer = builder.setBolt("ipZoneJoin", ipZoneDataJoinBolt, numOfIPZoneJoinTasks).setNumTasks(numOfIPZoneJoinTasks); ipZoneDataJoinBoltDeclarer.fieldsGrouping("sensitivityJoin", new Fields("user")); StormStreamSink sinkBolt = environment.getStreamSink("hdfs_audit_log_stream",config); -BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks); +BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks).setNumTasks(numOfSinkTasks); kafkaBoltDeclarer.fieldsGrouping("ipZoneJoin", new Fields("user")); return builder.createTopology();
[32/50] incubator-eagle git commit: [EAGLE-622] : fix the datasource name to topic name caused unit test failure
[EAGLE-622] : fix the datasource name to topic name caused unit test failure Author: ralphsu This closes #512 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7279ee71 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7279ee71 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7279ee71 Branch: refs/heads/master Commit: 7279ee711587a4c6364e324002baa1daea089270 Parents: 00e92d9 Author: Ralph, SuAuthored: Fri Oct 14 20:23:39 2016 +0800 Committer: Ralph, Su Committed: Fri Oct 14 20:30:51 2016 +0800 -- .../apache/alert/coordinator/SchedulerTest.java | 48 ++-- .../engine/topology/CorrelationSpoutTest.java | 4 +- 2 files changed, 16 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7279ee71/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java index d0b7a09..1bfdd7b 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java @@ -16,27 +16,16 @@ */ package org.apache.alert.coordinator; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.CollectionType; +import com.fasterxml.jackson.databind.type.SimpleType; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; import org.apache.alert.coordinator.mock.TestTopologyMgmtService; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.Pair; -import org.apache.eagle.alert.coordination.model.AlertBoltSpec; -import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata; -import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue; -import org.apache.eagle.alert.coordination.model.RouterSpec; -import org.apache.eagle.alert.coordination.model.ScheduleState; -import org.apache.eagle.alert.coordination.model.SpoutSpec; -import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata; -import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy; -import org.apache.eagle.alert.coordination.model.StreamRouterSpec; -import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata; -import org.apache.eagle.alert.coordination.model.WorkSlot; +import org.apache.eagle.alert.coordination.model.*; import org.apache.eagle.alert.coordination.model.internal.MonitoredStream; import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment; import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue; @@ -48,26 +37,17 @@ import org.apache.eagle.alert.coordinator.model.AlertBoltUsage; import org.apache.eagle.alert.coordinator.model.GroupBoltUsage; import org.apache.eagle.alert.coordinator.model.TopologyUsage; import org.apache.eagle.alert.coordinator.provider.InMemScheduleConext; -import org.apache.eagle.alert.engine.coordinator.PolicyDefinition; +import org.apache.eagle.alert.engine.coordinator.*; import org.apache.eagle.alert.engine.coordinator.PolicyDefinition.Definition; -import org.apache.eagle.alert.engine.coordinator.Publishment; -import org.apache.eagle.alert.engine.coordinator.StreamColumn; import org.apache.eagle.alert.engine.coordinator.StreamColumn.Type; -import org.apache.eagle.alert.engine.coordinator.StreamDefinition; -import org.apache.eagle.alert.engine.coordinator.StreamPartition; -import org.apache.eagle.alert.engine.coordinator.StreamSortSpec; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.CollectionType; -import com.fasterxml.jackson.databind.type.SimpleType; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.util.*; /** * @since Apr 22, 2016 @@ -130,7 +110,7 @@ public class SchedulerTest {
[33/50] incubator-eagle git commit: [EAGLE-562] added metrics associated with virtual-cores involved since hadoop 2.7.x
[EAGLE-562] added metrics associated with virtual-cores involved since hadoop 2.7.x add 4 virtual-cores related metrics involved since hadoop 2.7.x to ClusterMetrics - totalVirtualCores, reservedVirtualCores, availableVirtualCores, allocatedVirtualCores - in this commit, only totalVirtualCores is included to be listened Author: anyway1021 <m...@apache.org> Closes #494 from anyway1021/EAGLE-562. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/1a74b09b Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/1a74b09b Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/1a74b09b Branch: refs/heads/master Commit: 1a74b09b714412d023c99c4fbf6498cc16984657 Parents: 7279ee7 Author: anyway1021 <m...@apache.org> Authored: Sat Oct 15 13:28:52 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Sat Oct 15 13:28:52 2016 +0800 -- .../queue/common/HadoopClusterConstants.java| 1 + .../crawler/ClusterMetricsParseListener.java| 1 + .../queue/model/cluster/ClusterMetrics.java | 41 3 files changed, 43 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1a74b09b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java -- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java index 9318a35..c924455 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java +++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java @@ -44,6 +44,7 @@ public class HadoopClusterConstants { public static final String HADOOP_CLUSTER_TOTAL_MEMORY = "hadoop.cluster.totalmemory"; public static final String HADOOP_CLUSTER_AVAILABLE_MEMORY = "hadoop.cluster.availablememory"; public static final String HADOOP_CLUSTER_RESERVED_MEMORY = "hadoop.cluster.reservedmemory"; +public static final String HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES = "hadoop.cluster.totalvirtualcores"; // metrics from scheduler info public static final String HADOOP_CLUSTER_CAPACITY = "hadoop.cluster.capacity"; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1a74b09b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java -- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java index de93d1e..ed78465 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java +++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java @@ -83,6 +83,7 @@ public class ClusterMetricsParseListener { createMetric(MetricName.HADOOP_CLUSTER_TOTAL_MEMORY, currentTimestamp, metrics.getTotalMB(), AggregateFunc.MAX); createMetric(MetricName.HADOOP_CLUSTER_AVAILABLE_MEMORY, currentTimestamp, metrics.getAvailableMB(), AggregateFunc.AVG); createMetric(MetricName.HADOOP_CLUSTER_RESERVED_MEMORY, currentTimestamp, metrics.getReservedMB(), AggregateFunc.AVG); +createMetric(MetricName.HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES, currentTimestamp, metrics.getTotalVirtualCores(), AggregateFunc.MAX); } public void flush() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/1a74b09b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/cluster/ClusterMetrics.java -- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/cluster/ClusterMetrics.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/cluster/ClusterMetrics.java index 8bf8917..aca67dc 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/cluster/ClusterMetrics.java +++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/cluster/ClusterMetrics.java @@ -44,6 +44,15 @@ public class ClusterMetrics { private int rebootedNodes; private int activeNodes; +// ava
[08/50] incubator-eagle git commit: [EAGLE-601] Change updating policy status API as POST /metadata/policies/{policyId}/status/{status}
[EAGLE-601] Change updating policy status API as POST /metadata/policies/{policyId}/status/{status} Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/757f510a Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/757f510a Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/757f510a Branch: refs/heads/master Commit: 757f510a14a2c79b4d20b4c156aaf079cc1683fe Parents: 9205433 Author: Hao Chen <h...@apache.org> Authored: Tue Oct 11 11:57:33 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Oct 11 11:57:33 2016 +0800 -- .../apache/eagle/service/metadata/resource/MetadataResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/757f510a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java index 0bf31df..d3d41ea 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/metadata/resource/MetadataResource.java @@ -236,8 +236,8 @@ public class MetadataResource { } @Path("/policies/{policyId}/status/{status}") -@PUT -public OpResult updatePolicyStatusByID(@PathParam("policyId") String policyId, @PathParam("status")PolicyDefinition.PolicyStatus status) { +@POST +public OpResult updatePolicyStatusByID(@PathParam("policyId") String policyId, @PathParam("status") PolicyDefinition.PolicyStatus status) { OpResult result = new OpResult(); try { PolicyDefinition policyDefinition = getPolicyByID(policyId);
[39/50] incubator-eagle git commit: [EAGLE-615] Jsoup parse hive sql return String without line break "\n"
[EAGLE-615] Jsoup parse hive sql return String without line break "\n" EAGLE-615 Jsoup parse hive sql return String without line break "\n" - ADD "doc.outputSettings().prettyPrint(false);", get element value via getWholeText() not text() Author: chitin <chitin1...@gmail.com> Closes #499 from chitin/EAGLE615. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a710082d Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a710082d Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a710082d Branch: refs/heads/master Commit: a710082d486e10b4732c00a06dd367dc556df60a Parents: a6bc0a5 Author: chitin <chitin1...@gmail.com> Authored: Mon Oct 17 11:42:27 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Oct 17 11:42:27 2016 +0800 -- .../hive/jobrunning/HiveJobFetchSpout.java | 35 +++- 1 file changed, 26 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a710082d/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobFetchSpout.java -- diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobFetchSpout.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobFetchSpout.java index c0673b3..af4599b 100644 --- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobFetchSpout.java +++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobFetchSpout.java @@ -22,6 +22,7 @@ import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.tuple.Fields; +import org.apache.commons.lang.StringUtils; import org.apache.eagle.dataproc.impl.storm.ValuesArray; import org.apache.eagle.jpm.util.*; import org.apache.eagle.jpm.util.jobrecover.RunningJobManager; @@ -35,12 +36,14 @@ import org.apache.eagle.security.hive.config.RunningJobCrawlConfig; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.map.ObjectMapper; import org.jsoup.Jsoup; +import org.jsoup.nodes.TextNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; + import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import java.io.InputStream; @@ -62,6 +65,7 @@ public class HiveJobFetchSpout extends BaseRichSpout { private Long lastFinishAppTime; private RunningJobManager runningJobManager; private int partitionId; + static { OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); } @@ -91,22 +95,22 @@ public class HiveJobFetchSpout extends BaseRichSpout { // sanity verify 0<=partitionId<=numTotalPartitions-1 if (partitionId < 0 || partitionId > crawlConfig.controlConfig.numTotalPartitions) { throw new IllegalStateException("partitionId should be less than numTotalPartitions with partitionId " + -partitionId + " and numTotalPartitions " + crawlConfig.controlConfig.numTotalPartitions); +partitionId + " and numTotalPartitions " + crawlConfig.controlConfig.numTotalPartitions); } Class partitionerCls = crawlConfig.controlConfig.partitionerCls; try { this.jobFilter = new JobIdFilterByPartition(partitionerCls.newInstance(), -crawlConfig.controlConfig.numTotalPartitions, partitionId); +crawlConfig.controlConfig.numTotalPartitions, partitionId); } catch (Exception e) { LOG.error("failing instantiating job partitioner class " + partitionerCls.getCanonicalName()); throw new IllegalStateException(e); } this.collector = collector; this.runningJobManager = new RunningJobManager(crawlConfig.zkStateConfig.zkQuorum, -crawlConfig.zkStateConfig.zkSessionTimeoutMs, -crawlConfig.zkStateConfig.zkRetryTimes, -crawlConfig.zkStateConfig.zkRetryInterval, -crawlConfig.zkStateConfig.zkRoot); +crawlConfig.zkStateConfig.zkSessionTimeoutMs, +crawlConfig.zkStateConfig.zkRetryTimes, +crawlConfig.zkStateConfig.zkRetryInterval, +crawlConfig.zkStateConfig.zkRoot); this.lastFinishAppTime = th
[41/50] incubator-eagle git commit: [EAGLE-599] improve application config
[EAGLE-599] improve application config * Support JSON import * UI adjust - separate the general fields & merge env and config tab Author: zombieJCloses #519 from zombieJ/EAGLE-599. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ecda7d5a Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ecda7d5a Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ecda7d5a Branch: refs/heads/master Commit: ecda7d5abfe30125f9c9b7ee3ebf8e8a49ce054e Parents: eda6e58 Author: zombieJ Authored: Mon Oct 17 15:41:24 2016 +0800 Committer: zombieJ Committed: Mon Oct 17 15:41:24 2016 +0800 -- .../app/dev/partials/integration/site.html | 105 +- .../app/dev/public/js/ctrls/integrationCtrl.js | 108 +++ 2 files changed, 164 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ecda7d5a/eagle-server/src/main/webapp/app/dev/partials/integration/site.html -- diff --git a/eagle-server/src/main/webapp/app/dev/partials/integration/site.html b/eagle-server/src/main/webapp/app/dev/partials/integration/site.html index db6b7c7..ce0cddd 100644 --- a/eagle-server/src/main/webapp/app/dev/partials/integration/site.html +++ b/eagle-server/src/main/webapp/app/dev/partials/integration/site.html @@ -21,7 +21,7 @@ Site - + Description @@ -48,7 +48,7 @@ Version Description - Actions + Actions @@ -84,7 +84,7 @@ - Install + Install Application @@ -207,8 +207,7 @@ Installation - Environment - Configuration + Setting @@ -230,31 +229,85 @@ - - - Mode - - CLUSTER - LOCAL - + + + + + + Environment + + + + + Mode + + CLUSTER + LOCAL + + + + jar Path + + - - jar Path - + + + + + Config By JSON + +
[16/50] incubator-eagle git commit: EAGLE-602: Exception that Spec Version [xxx] of AlertBolt is newer than Stream Event Version [null]
EAGLE-602: Exception that Spec Version [xxx] of AlertBolt is newer than Stream Event Version [null] Auhtor: Zeng, Bryant Reviewer: ralphsu This closes #486 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/74c3cbb5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/74c3cbb5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/74c3cbb5 Branch: refs/heads/master Commit: 74c3cbb5b1653974870e805b717987eb75e5113d Parents: 3301c9d Author: mizengAuthored: Mon Oct 10 17:58:54 2016 +0800 Committer: Ralph, Su Committed: Thu Oct 13 15:26:19 2016 +0800 -- .../org/apache/eagle/alert/engine/runner/AlertBolt.java | 11 ++- .../engine/serialization/impl/StreamEventSerializer.java | 6 ++ 2 files changed, 12 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/74c3cbb5/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java index f437d43..fbee580 100755 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java @@ -107,11 +107,12 @@ public class AlertBolt extends AbstractStreamBolt implements AlertBoltSpecListen try { PartitionedEvent pe = deserialize(input.getValueByField(AlertConstants.FIELD_0)); String streamEventVersion = pe.getEvent().getMetaVersion(); -if (streamEventVersion != null && !streamEventVersion.equals(specVersion)) { -if (streamEventVersion == null) { -// if stream event version is null, need to initialize it -pe.getEvent().setMetaVersion(specVersion); -} else if (specVersion != null && streamEventVersion != null + +if (streamEventVersion == null) { +// if stream event version is null, need to initialize it +pe.getEvent().setMetaVersion(specVersion); +} else if (streamEventVersion != null && !streamEventVersion.equals(specVersion)) { +if (specVersion != null && streamEventVersion != null && specVersion.contains("spec_version_") && streamEventVersion.contains("spec_version_")) { // check if specVersion is older than stream_event_version // Long timestamp_of_specVersion = Long.valueOf(specVersion.split("spec_version_")[1]); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/74c3cbb5/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java index d7119db..8ffcb83 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java @@ -89,6 +89,12 @@ public class StreamEventSerializer implements Serializer { String metaVersionStreamId = dataInput.readUTF(); String streamId = metaVersionStreamId.split("/")[1]; String metaVersion = metaVersionStreamId.split("/")[0]; +// sometimes metaVersionStreamId will be "null/id", then metaVersion will be "null" rather than null +// need to handle it for future use +if (metaVersion.equals("null")) { +metaVersion = null; +} + event.setStreamId(streamId); event.setMetaVersion(metaVersion);
[01/50] incubator-eagle git commit: [EAGLE-598] Create 'ui' building profile and enable Eagle Web UI Building by default [Forced Update!]
Repository: incubator-eagle Updated Branches: refs/heads/master 0474d5916 -> 6dbdb4f72 (forced update) [EAGLE-598] Create 'ui' building profile and enable Eagle Web UI Building by default Create "ui" building profile and enable Eagle Web UI Building by default mvn clean install -DskipTests Could also disable UI building with: mvn clean install -DskipTests -P!ui Or (if "!" is reserved command in OS) mvn clean install -DskipTests -P\!ui Author: Hao Chen <h...@apache.org> Closes #482 from haoch/EAGLE-598. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0277ff75 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0277ff75 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0277ff75 Branch: refs/heads/master Commit: 0277ff75e10620cdfc17d01823a37719af63995b Parents: 1731612 Author: Hao Chen <h...@apache.org> Authored: Mon Oct 10 15:54:52 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Mon Oct 10 15:54:52 2016 +0800 -- eagle-server/pom.xml| 64 +--- .../server/managedtask/ApplicationTask.java | 4 +- eagle-server/ui-build.sh| 5 +- pom.xml | 6 ++ 4 files changed, 53 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0277ff75/eagle-server/pom.xml -- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index 99de507..aa190e4 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -157,13 +157,43 @@ +ui + +true + + + + +org.codehaus.mojo +exec-maven-plugin + + +exec-ui-install +generate-sources + +exec + + +bash + + ${basedir}/ui-build.sh + + + + + + + + + + dev true @@ -223,28 +253,16 @@ - - - - - - - - - - - - - - - - - - - - - - + + +org.apache.maven.plugins +maven-checkstyle-plugin + +true +true + + + src/main/webapp/app/ui http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0277ff75/eagle-server/src/main/java/org/apache/eagle/server/managedtask/ApplicationTask.java -- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/managedtask/ApplicationTask.java b/eagle-server/src/main/java/org/apache/eagle/server/managedtask/ApplicationTask.java index d1173a8..b3a1b95 100644 --- a/eagle-server/src/main/java/org/apache/eagle/server/managedtask/ApplicationTask.java +++ b/eagle-server/src/main/java/org/apache/eagle/server/managedtask/ApplicationTask.java @@ -23,10 +23,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ApplicationTask implements Managed { -private final static Logger LOG = LoggerFactory.getLogger(ApplicationTask.class); +private static final Logger LOG = LoggerFactory.getLogger(ApplicationTask.class); private final AbstractScheduledService service; -public ApplicationTask(AbstractScheduledService service){ +public ApplicationTask(AbstractScheduledService service) { this.service = service; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0277ff75/eagle-server/ui-build.sh -- diff
incubator-eagle git commit: [EAGLE-635] Refactor policy parser and validator for richer plan details and better performance
Repository: incubator-eagle Updated Branches: refs/heads/master adc2ba900 -> 6dbdb4f72 [EAGLE-635] Refactor policy parser and validator for richer plan details and better performance ## Changes * Refactor policy parser and validator for richer plan details and better performance * Decouple PolicyExecutionPlan and PolicyValidation ## API * Parse API ~~~ POST /metadata/policies/parse Accept-Type: text from HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX#window.timeBatch(2 min) select cmd, user, count() as total_count group by cmd,user insert into HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT" ~~~ * Validation API ~~~ POST /metadata/policies/validate Accept-Type: application/json { "name": "hdfsPolicy", "description": "hdfsPolicy", "inputStreams": [ "HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX" ], "outputStreams": [ "HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT" ], "definition": { "type": "siddhi", "value": "from HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX#window.timeBatch(2 min) select cmd, user, count() as total_count group by cmd,user insert into HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX_OUT " }, "partitionSpec": [ { "streamId": "hdfs_audit_log_enriched_stream", "type": "GROUPBY", "columns" : [ "cmd" ] } ], "parallelismHint": 2 } ~~~ Author: Hao Chen <h...@apache.org> Closes #529 from haoch/RefactorPolicyValidator. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6dbdb4f7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6dbdb4f7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6dbdb4f7 Branch: refs/heads/master Commit: 6dbdb4f72ec5adf89f4f7a13bf766e01e3ff6705 Parents: adc2ba9 Author: Hao Chen <h...@apache.org> Authored: Wed Oct 19 10:28:09 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Wed Oct 19 10:28:09 2016 +0800 -- .../conf/sandbox-userprofile-scheduler.conf | 2 +- .../eagle-alert/alert-common/pom.xml| 8 - .../eagle-alert/alert-engine/pom.xml| 10 +- .../evaluator/impl/SiddhiDefinitionAdapter.java | 13 + .../alert-metadata-service/pom.xml | 13 - .../metadata/resource/MetadataResource.java | 10 +- .../metadata/resource/PolicyCompiler.java | 235 +++ .../metadata/resource/PolicyExecutionPlan.java | 100 .../metadata/resource/PolicyParseResult.java| 65 + .../metadata/resource/PolicyValidation.java | 97 .../resource/PolicyValidationResult.java| 76 ++ .../metadata/resource/PolicyValidator.java | 124 -- .../metadata/resource/PolicyCompilerTest.java | 195 +++ .../metadata/resource/PolicyValidatorTest.java | 187 --- 14 files changed, 699 insertions(+), 436 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6dbdb4f7/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf -- diff --git a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf b/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf index f54162d..4593a7e 100644 --- a/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf +++ b/eagle-assembly/src/main/conf/sandbox-userprofile-scheduler.conf @@ -63,4 +63,4 @@ akka { # Filter of log events that is used by the LoggingAdapter before # publishing log events to the eventStream. logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6dbdb4f7/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml index 57a0157..fd3eccd 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/pom.xml @@ -63,10 +63,6 @@ slf4j-api -org.slf4j -slf4j-log4j12 - - org.apache.curator curator-framework @@ -82,10 +78,6 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - - - joda-time joda-time
incubator-eagle git commit: [UI] Improve integeration page
Repository: incubator-eagle Updated Branches: refs/heads/master c3bde0194 -> adc2ba900 [UI] Improve integeration page Be sure to do all of the following to help us incorporate your contribution quickly and easily: - [x] Make sure the PR title is formatted like: `[EAGLE-] Description of pull request` - [x] Make sure tests pass via `mvn clean verify`. (Even better, enable Travis-CI on your fork and ensure the whole test matrix passes). - [x] Replace `` in the title with the actual Jira issue number, if there is one. - [x] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt). --- Author: Hao Chen <h...@apache.org> Closes #524 from haoch/ImproveIntegerationPage. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/adc2ba90 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/adc2ba90 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/adc2ba90 Branch: refs/heads/master Commit: adc2ba900a2d29352e96489ad48addde17bedccf Parents: c3bde01 Author: Hao Chen <h...@apache.org> Authored: Tue Oct 18 19:31:48 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Oct 18 19:31:48 2016 +0800 -- ...e.alert.app.AlertUnitTopologyAppProvider.xml | 83 +++- eagle-server/pom.xml| 10 +++ .../app/dev/partials/integration/site.html | 50 ++-- .../app/dev/public/js/ctrls/integrationCtrl.js | 8 +- eagle-server/ui-build.sh| 0 5 files changed, 87 insertions(+), 64 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/adc2ba90/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml -- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml index b3e777c..6ef96c7 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -20,110 +20,123 @@ AlertUnitTopologyApp Alert Unit Topology Application 0.5.0-incubating -org.apache.eagle.alert.app.AlertUnitTopologyApp -/apps/alert workers -workers +Worker Number 1 -number of topology workers +Storm parallelism worker number +false topology.numOfSpoutTasks -topology.numOfSpoutTasks +Source Tasks Number 1 -number of spout tasks +Number of spout tasks +false topology.numOfRouterBolts -topology.numOfRouterBolts +Route Tasks Number 4 -number of router tasks +Number of router tasks +false topology.numOfAlertBolts -topology.numOfAlertBolts +Evaluator Tasks Number 10 -number of alert tasks +Number of alert tasks +false topology.numOfPublishTasks -topology.numOfPublishTasks +Publisher Tasks Number 1 -number of publish tasks +Number of publish tasks +false topology.messageTimeoutSecs -topology.messageTimeoutSecs +Message Timeout Seconds 3600 -number of tuple timeout in seconds +Number of tuple timeout in seconds +false spout.kafkaBrokerZkQuorum -spout.kafkaBrokerZkQuorum +Kafka Zookeeper Quorum localhost:2181 -zookeeper quorum for spout to consume data +Zookeeper quorum of kafka broker for spout to consume data +true spout.kafkaBrokerZkBasePath -spout.kafkaBrokerZkBasePath +Kafka Zookeeper Root /brokers -zk znode path for kafka brokers +Zookeeper znode path for kafka broke