mooli tayer has uploaded a new change for review. Change subject: Tools: This change introduces the concept of configuration subscribers ......................................................................
Tools: This change introduces the concept of configuration subscribers Event subscribers can come from both the database and the configuration file, See ovirt-engine-notifier.conf.in for a detailed explanation. The EventSubscribersProvider was introduces to abstract the src of a subscribers list. A new EventFilter, RegexpEventFilter is used for configuration subscribers. Change-Id: Id863557f3902ab18f5dac7a822ca55ad0c96dda5 Signed-off-by: Mooli Tayer <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AddEventSubscriptionCommand.java A backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/RegexpEventFilter.java M backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java M backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/dao/EventsManager.java A backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/ConfigurationEventsSubscribersProvider.java A backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/DBEventsSubscribersProvider.java A backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/EventSubscribersProvider.java M backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java M packaging/services/ovirt-engine-notifier/ovirt-engine-notifier.conf.in 9 files changed, 302 insertions(+), 42 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/54/23154/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AddEventSubscriptionCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AddEventSubscriptionCommand.java index ba9f714..4a7f203 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AddEventSubscriptionCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AddEventSubscriptionCommand.java @@ -75,7 +75,7 @@ /** * Determines whether [is already subscribed] [the specified subscriptions]. - * + * * @param subscriptions * The subscriptions. * @param subscriberId diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/RegexpEventFilter.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/RegexpEventFilter.java new file mode 100644 index 0000000..b46f671 --- /dev/null +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/businessentities/RegexpEventFilter.java @@ -0,0 +1,17 @@ +package org.ovirt.engine.core.common.businessentities; + +import java.util.regex.Pattern; + +public class RegexpEventFilter implements EventFilter { + + private Pattern pattern; + + public void setPattern(Pattern pattern) { + this.pattern = pattern; + } + + @Override + public boolean isSubscribed(AuditLogEvent event) { + return pattern.matcher(event.getLogTypeName()).matches(); + } +} diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java index e157396..ba11c76 100644 --- a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java @@ -2,18 +2,16 @@ import java.net.ConnectException; import java.sql.SQLException; -import java.util.Collections; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.ovirt.engine.core.common.EventNotificationMethod; import org.ovirt.engine.core.common.businessentities.AuditLogEvent; import org.ovirt.engine.core.common.businessentities.AuditLogEventSubscriber; -import org.ovirt.engine.core.common.businessentities.UPDownEventFilter; import org.ovirt.engine.core.notifier.dao.EventsManager; -import org.ovirt.engine.core.notifier.utils.NotificationMethodsMapper; +import org.ovirt.engine.core.notifier.subscribers.ConfigurationEventsSubscribersProvider; +import org.ovirt.engine.core.notifier.subscribers.DBEventsSubscribersProvider; +import org.ovirt.engine.core.notifier.subscribers.EventSubscribersProvider; import org.ovirt.engine.core.notifier.utils.NotificationProperties; import org.ovirt.engine.core.notifier.sender.*; @@ -25,18 +23,29 @@ private static final Logger log = Logger.getLogger(NotificationService.class); - private NotificationProperties prop = null; + private final NotificationProperties prop; + private final EventsManager eventsManager; - private NotificationMethodsMapper notificationMethodsMapper; - private List<AuditLogEventSubscriber> failedQueriesEventSubscribers = Collections.emptyList(); + + private final NotificationMethodsMapper notificationMethodsMapper; + + private final List<EventSubscribersProvider> eventSubscribersProviders = new ArrayList<>(); + + private final List<EventSubscribersProvider> dbDownEventSubscribersProviders = new ArrayList<>(); + private int failedQueries = 0; public NotificationService(NotificationProperties prop) throws NotificationServiceException { this.prop = prop; - notificationMethodsMapper = new NotificationMethodsMapper(prop); + this.notificationMethodsMapper = new NotificationMethodsMapper(prop); this.eventsManager = new EventsManager(); + + final DBEventsSubscribersProvider dbEventsSubscribersProvider = new DBEventsSubscribersProvider(); + dbEventsSubscribersProvider.setManager(eventsManager); + this.eventSubscribersProviders.add(dbEventsSubscribersProvider); + this.eventSubscribersProviders.add(new ConfigurationEventsSubscribersProvider(prop)); + this.dbDownEventSubscribersProviders.add(new ConfigurationEventsSubscribersProvider(prop)); markOldEventsAsProcessed(); - failedQueriesEventSubscribers = initFailedQueriesEventSubscribers(); } private void markOldEventsAsProcessed() { @@ -56,7 +65,7 @@ public void run() { try { log.debug("Start event notification service iteration"); - List<AuditLogEventSubscriber> eventSubscribers = eventsManager.getAuditLogEventSubscribers(); + List<AuditLogEventSubscriber> eventSubscribers = getEventSubscribers(false); List<AuditLogEvent> events = eventsManager.getAuditLogEvents(true); log.debug(String.format("%d unprocessed events read.", events.size())); distributeEvents(events, eventSubscribers); @@ -82,6 +91,22 @@ log.debug(deletedRecords + " records were deleted from \"event_notification_hist\" table."); } } + } + + private List<AuditLogEventSubscriber> getEventSubscribers(boolean dbDown) { + List<AuditLogEventSubscriber> eventSubscribers = new ArrayList<>(); + if (dbDown) { + for (EventSubscribersProvider eventSubscribersProvider : dbDownEventSubscribersProviders) { + eventSubscribers.addAll(eventSubscribersProvider.provide()); + } + } else { + for (EventSubscribersProvider eventSubscribersProvider : eventSubscribersProviders) { + eventSubscribers.addAll(eventSubscribersProvider.provide()); + } + } + log.debug(String.format("%d subscribers read from %d providers.", eventSubscribers.size(), + dbDown ? dbDownEventSubscribersProviders.size() : eventSubscribersProviders.size())); + return eventSubscribers; } private void distributeEvents(List<AuditLogEvent> events, List<AuditLogEventSubscriber> eventSubscribers) @@ -117,7 +142,7 @@ private void distributeDbDownEvent() { if (failedQueries == 0) { try { - distributeEvents(eventsManager.getAuditLogEvents(true), failedQueriesEventSubscribers); + distributeEvents(eventsManager.getAuditLogEvents(true), getEventSubscribers(true)); } catch (Exception e) { log.error("Failed to dispatch query failure email message", e); // Don't rethrow. we don't want to mask the original query exception. @@ -129,22 +154,6 @@ failedQueriesNotificationThreshold = 1; } failedQueries = (failedQueries + 1) % failedQueriesNotificationThreshold; - } - - private List<AuditLogEventSubscriber> initFailedQueriesEventSubscribers() { - List<AuditLogEventSubscriber> failureSubscribers = new LinkedList<>(); - String subscribers = prop.getProperty(NotificationProperties.FAILED_QUERIES_NOTIFICATION_RECIPIENTS); - if (!StringUtils.isEmpty(subscribers)) { - for (String email : subscribers.split(",")) { - AuditLogEventSubscriber dbDownSubscriber = new AuditLogEventSubscriber(); - dbDownSubscriber.setEventNotificationMethod(EventNotificationMethod.EMAIL); - dbDownSubscriber.setMethodAddress(StringUtils.strip(email)); - final UPDownEventFilter eventFilter = new UPDownEventFilter(); - eventFilter.setEventUpName("DATABASE_UNREACHABLE"); - dbDownSubscriber.setEventFilter(eventFilter); - } - } - return failureSubscribers; } } diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/dao/EventsManager.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/dao/EventsManager.java index cee66ae..8a2d5f8 100644 --- a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/dao/EventsManager.java +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/dao/EventsManager.java @@ -7,7 +7,7 @@ import org.ovirt.engine.core.common.businessentities.AuditLogEvent; import org.ovirt.engine.core.common.businessentities.AuditLogEventSubscriber; import org.ovirt.engine.core.common.businessentities.AuditLogEventType; -import org.ovirt.engine.core.common.businessentities.UPDownEventFilter; +import org.ovirt.engine.core.common.businessentities.UpDownEventFilter; import org.ovirt.engine.core.compat.Guid; import org.ovirt.engine.core.notifier.NotificationServiceException; import org.ovirt.engine.core.notifier.sender.EventSenderResult; @@ -159,7 +159,7 @@ newSubscriber.setMethodAddress(email); newSubscriber.setSubscriberId(Guid.createGuidFromStringDefaultEmpty(rs.getString("subscriber_id"))); newSubscriber.setUsername(rs.getString("username")); - UPDownEventFilter eventFilter = new UPDownEventFilter(); + UpDownEventFilter eventFilter = new UpDownEventFilter(); eventFilter.setEventUpName(rs.getString("event_up_name")); eventFilter.setEventDownName(rs.getString("event_down_name")); newSubscriber.setEventFilter(eventFilter); diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/ConfigurationEventsSubscribersProvider.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/ConfigurationEventsSubscribersProvider.java new file mode 100644 index 0000000..e877045 --- /dev/null +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/ConfigurationEventsSubscribersProvider.java @@ -0,0 +1,73 @@ +package org.ovirt.engine.core.notifier.subscribers; + +import org.apache.commons.lang.StringUtils; +import org.ovirt.engine.core.common.EventNotificationMethod; +import org.ovirt.engine.core.common.businessentities.AuditLogEventSubscriber; +import org.ovirt.engine.core.common.businessentities.RegexpEventFilter; +import org.ovirt.engine.core.common.businessentities.UpDownEventFilter; +import org.ovirt.engine.core.notifier.utils.NotificationProperties; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +public class ConfigurationEventsSubscribersProvider implements EventSubscribersProvider { + + private NotificationProperties prop; + + private List<AuditLogEventSubscriber> cache; + + public ConfigurationEventsSubscribersProvider(NotificationProperties prop) { + this.prop = prop; + } + + @Override + public List<AuditLogEventSubscriber> provide() { + if (cache == null) { + readConfiguration(); + } + return cache; + } + + private void readConfiguration() { + Map<String, AuditLogEventSubscriber> subscribersByName = new HashMap<>(); + final Map<String, String> properties = prop.getProperties(); + for (Map.Entry<String, String> entry : properties.entrySet()) { + final String key = entry.getKey(); + if (key.startsWith(NotificationProperties.SUBSCRIBERS_PREFIX)) { + final String[] split = key.split("."); + final String methodName = split[1]; + final String propType = split[2]; + final String name = split[3]; + AuditLogEventSubscriber newSubscriber = subscribersByName.get(name); + if (newSubscriber == null) { + newSubscriber = new AuditLogEventSubscriber(); + subscribersByName.put(name, newSubscriber); + } + newSubscriber.setEventNotificationMethod(EventNotificationMethod.valueOf(methodName)); + if (NotificationProperties.EVENT_REGEXP.equals(propType)) { + final RegexpEventFilter eventFilter = new RegexpEventFilter(); + eventFilter.setPattern(Pattern.compile(entry.getValue())); + newSubscriber.setEventFilter(eventFilter); + } else { // NotificationProperties.METHOD_ADDRESS + newSubscriber.setMethodAddress(entry.getValue()); + } + } else if (key.equals(NotificationProperties.FAILED_QUERIES_NOTIFICATION_RECIPIENTS)) { + final String value = entry.getValue(); + if (!StringUtils.isEmpty(value)) { + for (String email : value.split(",")) { + AuditLogEventSubscriber dbDownSubscriber = new AuditLogEventSubscriber(); + dbDownSubscriber.setEventNotificationMethod(EventNotificationMethod.EMAIL); + dbDownSubscriber.setMethodAddress(StringUtils.strip(email)); + final UpDownEventFilter eventFilter = new UpDownEventFilter(); + eventFilter.setEventUpName("DATABASE_UNREACHABLE"); + dbDownSubscriber.setEventFilter(eventFilter); + } + } + } + } + cache = new LinkedList<>(subscribersByName.values()); + } +} diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/DBEventsSubscribersProvider.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/DBEventsSubscribersProvider.java new file mode 100644 index 0000000..d9c706e --- /dev/null +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/DBEventsSubscribersProvider.java @@ -0,0 +1,24 @@ +package org.ovirt.engine.core.notifier.subscribers; + +import org.ovirt.engine.core.common.businessentities.AuditLogEventSubscriber; +import org.ovirt.engine.core.notifier.dao.EventsManager; + +import java.util.Collections; +import java.util.List; + +public class DBEventsSubscribersProvider implements EventSubscribersProvider { + + EventsManager manager; + + public void setManager(EventsManager manager) { + this.manager = manager; + } + + @Override + public List<AuditLogEventSubscriber> provide() { + if (manager != null){ + return manager.getAuditLogEventSubscribers(); + } + return Collections.emptyList(); + } +} diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/EventSubscribersProvider.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/EventSubscribersProvider.java new file mode 100644 index 0000000..04ee9f1 --- /dev/null +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/subscribers/EventSubscribersProvider.java @@ -0,0 +1,12 @@ +package org.ovirt.engine.core.notifier.subscribers; + + +import org.ovirt.engine.core.common.businessentities.AuditLogEventSubscriber; + +import java.util.List; + +public interface EventSubscribersProvider { + + List<AuditLogEventSubscriber> provide(); + +} diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java index 54fcb50..c288dc5 100644 --- a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java @@ -1,9 +1,14 @@ package org.ovirt.engine.core.notifier.utils; import java.net.InetAddress; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.mail.internet.InternetAddress; import org.apache.commons.lang.StringUtils; +import org.ovirt.engine.core.common.EventNotificationMethod; import org.ovirt.engine.core.utils.LocalConfig; /** @@ -72,6 +77,14 @@ */ public static final String MAIL_SMTP_ENCRYPTION_TLS = "tls"; + /** + * Configuration file subscribers + */ + public static final String SUBSCRIBERS_PREFIX = "SUBSCRIBER."; + + public static final String METHOD_ADDRESS = "METHOD_ADDRESS"; + + public static final String EVENT_REGEXP = "EVENT_REGEXP"; // Default files for defaults and overridden values: private static String DEFAULTS_PATH = "/usr/share/ovirt-engine/conf/notifier.conf.defaults"; private static String VARS_PATH = "/etc/ovirt-engine/notifier/notifier.conf"; @@ -208,6 +221,84 @@ "'%s' must be set when SSL or TLS is enabled or when password is set", NotificationProperties.MAIL_USER)); } + + // validate Subscribers + Map<String, String> subscribers = new HashMap<>(); + Map<String, String> names = new HashMap<>(); + final Map<String, String> properties = getProperties(); + for (Map.Entry<String, String> entry : properties.entrySet()) { + final String key = entry.getKey(); + if (key.startsWith(NotificationProperties.SUBSCRIBERS_PREFIX)) { + final String[] split = key.split("."); + if (split.length != 3) { + throw new IllegalArgumentException( + String.format( + "'%s' is not a valid subscriber key. it must have 4 . delimited tokens.", + key)); + } + final String name = split[3]; + final String propType = split[2]; + final String methodName = split[1]; + final String val = subscribers.get(name); + if (!NotificationProperties.METHOD_ADDRESS.equals(propType) && + !NotificationProperties.EVENT_REGEXP.equals(propType)) { + throw new IllegalArgumentException( + String.format( + "'%s' is not a valid subscriber key. %s must be %s or %s.", + key, propType, NotificationProperties.METHOD_ADDRESS, + NotificationProperties.EVENT_REGEXP)); + + } + + EventNotificationMethod.valueOf(methodName); + + Pattern namePattern = + Pattern.compile("[a-zA-Z][a-zA-Z&&[0-9]]*"); + final Matcher nameMatcher = namePattern.matcher(name); + if (!nameMatcher.matches()) { + throw new IllegalArgumentException( + String.format( + "subscriber '%s' must appear in the configuration file exactly twice.", + name)); + } + if (val == null) { + subscribers.put(name, methodName + propType); + if (names.containsKey(name)) { + throw new IllegalArgumentException( + String.format( + "'%s' is an illegal subscriber name. it must start with a letter and contain only" + + " letters and digits", + name)); + } + names.put(name, null); + } else { + if (!val.startsWith(methodName)) { + throw new IllegalArgumentException( + String.format( + "subscriber '%s' is subscribed to two different notificationMethods.", + name)); + + } + if (val.endsWith(propType)) { + throw new IllegalArgumentException( + String.format( + "'%s' is defined twice for %s.", + propType, name)); + + } + subscribers.remove(name); + } + } + } + if (!subscribers.isEmpty()) { + final String name = subscribers.keySet().iterator().next(); + throw new IllegalArgumentException( + String.format( + "subscriber '%s' must appear in the configuration file exactly twice.", + name)); + + } + } /** diff --git a/packaging/services/ovirt-engine-notifier/ovirt-engine-notifier.conf.in b/packaging/services/ovirt-engine-notifier/ovirt-engine-notifier.conf.in index c835fcf..5ac9e22 100644 --- a/packaging/services/ovirt-engine-notifier/ovirt-engine-notifier.conf.in +++ b/packaging/services/ovirt-engine-notifier/ovirt-engine-notifier.conf.in @@ -43,6 +43,22 @@ # Interval (in seconds) between iterations of dispatching messages to subscribers. Default is 120 seconds. INTERVAL_IN_SECONDS=120 +# Amount of days to keep dispatched events on history table. If 0, events remain on history table for ever. +DAYS_TO_KEEP_HISTORY=0 + +# Send a notification email after first failure to fetch notifications, +# and then once every failedQueriesNotificationThreshold times. +# 0 or 1 means notify on each failure. +FAILED_QUERIES_NOTIFICATION_THRESHOLD=30 + +# Comma separated list of email recipients to be informed in case +# the notification service cannot connect to the DB. can be empty. +FAILED_QUERIES_NOTIFICATION_RECIPIENTS= + +#---------------------# +# EMAIL Notifications # +#---------------------# + # The SMTP mail server address. Required. MAIL_SERVER= @@ -69,9 +85,6 @@ # Specifies 'reply-to' address on sent mail in RFC822 format. MAIL_REPLY_TO= -# Amount of days to keep dispatched events on history table. If 0, events remain on history table. -DAYS_TO_KEEP_HISTORY=0 - # This parameter specifies how many days of old events are processed and sent # when the notifier starts. If set to 2, for example, the notifier will # process and send the events of the last two days, older events will just @@ -79,14 +92,35 @@ # messages will be sent at all during startup. DAYS_TO_SEND_ON_STARTUP=0 -# Send a notification email after first failure to fetch notifications, -# and then once every failedQueriesNotificationThreshold times. -# 0 or 1 means notify on each failure. -FAILED_QUERIES_NOTIFICATION_THRESHOLD=30 +#-------------# +# Subscribers # +#-------------# -# Comma separated list of email recipients to be informed in case -# the notification service cannot connect to the DB. can be empty. -FAILED_QUERIES_NOTIFICATION_RECIPIENTS= +# +# Configuration pairs in this section define audit_log event subscriptions. +# Events subscriptions used by ovirt-engine-notifier are a union of those listed here and those taken from +# the 'event_subscriber' table. two keys must be defined; +# SUBSCRIBER.[METHOD_NAME].METHOD_ADDRESS.[SUBSCRIBER_NAME]= +# SUBSCRIBER.[METHOD_NAME].EVENT_REGEXP.[SUBSCRIBER_NAME]= +# +# METHOD_NAME can be one of EMAIL, SNMP. +# SUBSCRIBER_NAME is an arbitrary name starting with a letter and containing letters digits and underscores. + every name must appear in this file exactly twice. +# METHOD_ADDRESS for email an RFC822 format address. For snmp a hostname[:port] pair. default snmp port is 162. +# EVENT_REGEXP note: unlike the 'event_subscriber' table where events are matched based on an 'event_up_name' and the +# matching 'event_down_name' from the 'event_map' table, here the matching is based on the regexp alone. +# In both cases the matching is against the 'audit_log' table's 'log_type_name' column. + +# Note: email subscribers can also be created in 'event_subscriber' using the ovirt-engine webadmin. + +# EXAMPLE: define an snmp subscriber named foo receiving traps to snmp_manager.example.com on port 162 on all events. +#SUBSCRIBER.SNMP.METHOD_ADDRESS.FOO=snmp_manager.example.com +#SUBSCRIBER.SNMP.EVENT_REGEXP.FOO=.* + +# EXAMPLE: define an email subscriber named bar receiving messages to [email protected] on vdc_stop and vdc_start events. +#[email protected] +#SUBSCRIBER.EMAIL.EVENT_REGEXP.BAR=VDC_START|VDC_STOP + #----------------------------------# # Engine Monitoring Configuration: # -- To view, visit http://gerrit.ovirt.org/23154 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id863557f3902ab18f5dac7a822ca55ad0c96dda5 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: mooli tayer <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
