Hi Jochen 

I installed Graylog2 via the OVA I downloaded off the Site.

It can fluctuate you see there are specials that run on the website which 
can cause massive amounts of traffic so it can be fine one week then within 
the span of 2 days I can get anything from 1million messages to 4 million 
messages. Common story is that it cant keep up with that influx of messages.

Grok patterns that I use. Please see attachment Grokpatternsfile.txt

Extractors please see attachment Extractors.txt

I have not setup dashboards or streams as of yet.

Purpose of Graylog is im trying to extract WAF logs (so if there is a 
better way of doing this please help.)

If there is more info you want please let me know and I do appreciate your 
help in advance.

Thanks 


On Monday, November 23, 2015 at 12:58:45 PM UTC+2, Jochen Schalanda wrote:
>
> Hi Matthew,
>
> to not duplicate the information you already gave on Twitter (
> https://twitter.com/Malfufi/status/668724729629556736):
>
>> heap 1.4G (changeability?) no local configuration has been made to 
>> optimize,85mill and CPU 100% RAM 100%
>
>
> The heap size of Graylog can be changed quite easily. How did you install 
> Graylog in the first place (OVA, DEB packages, RPM packages, tarball…)?
>
> "85 mill" what exactly? Messages per second/minute/hour/day/week/month? 
> Are you running any extractors (e. g. RegEx or Grok extractors) or defined 
> streams with relatively complicated regular expression rules?
>
>
> Cheers,
> Jochen
>
> On Monday, 23 November 2015 10:55:48 UTC+1, Matthew Simon wrote:
>>
>> Hi Guys 
>>
>>
>> I have a problem!
>>
>>
>> I receive large amounts of logs to my Graylog2 server and i feel that the 
>> server cant keep up with the incoming logs, Is there a way that I can 
>> optimize my configuration to handle large amounts of LOGS. 
>>
>>
>> Please see the image bellow.
>>
>>
>> Thanks in advance.
>>
>>
>>
>>
>> <https://lh3.googleusercontent.com/-n91LHSZ_HtQ/VlLhwvs-sKI/AAAAAAAAAqU/dXnmzSAQFCg/s1600/image1.png>
>>
>>
>>
>>

-- 
You received this message because you are subscribed to the Google Groups 
"Graylog Users" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to [email protected].
To view this discussion on the web visit 
https://groups.google.com/d/msgid/graylog2/5137a75c-31d7-4040-b0b1-18e836d191cd%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT 
\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*

QUOTEDSTRING 
(?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
QUOTEDSTRING 
(?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}

# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 
((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 
(?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME 
\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME}
IPORHOST (?:%{HOSTNAME}|%{IP})
HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})

# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?

# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
#URIPARAM 
\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?

# Months: January, Feb, 3, 03, 12, December
MONTH 
\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])

# Days: Monday, Tue, Thu, etc...
DAY 
(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)

# Years?
YEAR (?>\d\d){1,2}

# Time: HH:MM:SS
#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
# I'm still on the fence about using grok to perform the time match,
# since it's probably slower.

# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])

# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])

# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T 
]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}

# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG (?:[\w._/%-]+)
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}

# Shortcuts
QS %{QUOTEDSTRING}

# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} 
)?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} 
\[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: 
HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} 
(?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}

# Log Levels
LOGLEVEL 
([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
{
  "extractors": [
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^<(\\d.+)>"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "full_message",
      "target_field": "utm_logid",
      "title": "utm_logid"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "id=\"?([0-9.+-]+([0-9]+))\"?"
      },
      "extractor_type": "regex",
      "order": 1,
      "source_field": "full_message",
      "target_field": "utm_id",
      "title": "utm_id"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": 
"localip=\"?(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\"?"
      },
      "extractor_type": "regex",
      "order": 3,
      "source_field": "full_message",
      "target_field": "utm_localip",
      "title": "utm_localip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "size=\"?([0-9.+-]+([0-9]+))\"?"
      },
      "extractor_type": "regex",
      "order": 4,
      "source_field": "full_message",
      "target_field": "utm_size",
      "title": "utm_size"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "user=\"([a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 5,
      "source_field": "full_message",
      "target_field": "utm_user",
      "title": "utm_user"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "host=\"?(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\"?"
      },
      "extractor_type": "regex",
      "order": 6,
      "source_field": "full_message",
      "target_field": "utm_host",
      "title": "utm_host"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "method=\"([a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 7,
      "source_field": "full_message",
      "target_field": "utm_method",
      "title": "utm_method"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "statuscode=\"?([0-9.+-]+([0-9]+))\"?"
      },
      "extractor_type": "regex",
      "order": 8,
      "source_field": "full_message",
      "target_field": "utm_statuscode",
      "title": "utm_statuscode"
   },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "reason=\"([a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 9,
      "source_field": "full_message",
      "target_field": "utm_reason",
      "title": "utm_reason"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "srcip=\"?(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\"?"
      },
      "extractor_type": "regex",
      "order": 2,
      "source_field": "full_message",
      "target_field": "utm_srcip",
      "title": "utm_srcip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "referer=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 15,
      "source_field": "full_message",
      "target_field": "utm_referer",
      "title": "utm_referer"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "server=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 14,
      "source_field": "full_message",
      "target_field": "utm_server",
      "title": "utm_server"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "exceptions=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 11,
      "source_field": "full_message",
      "target_field": "utm_exception",
      "title": "utm_exception"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "extra=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 10,
      "source_field": "full_message",
      "target_field": "utm_extra",
      "title": "utm_extra"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "time=\"?([0-9.+-]+([0-9]+))\"?"
      },
      "extractor_type": "regex",
      "order": 12,
      "source_field": "full_message",
      "target_field": "utm_time",
      "title": "utm_time"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "url=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 13,
      "source_field": "full_message",
      "target_field": "utm_url",
      "title": "utm_url"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "cookie=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 16,
      "source_field": "full_message",
      "target_field": "utm_cookie",
      "title": "utm_cookie"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "set-cookie=\"?([/:?=a-zA-Z0-9._-]+)\"?"
      },
      "extractor_type": "regex",
      "order": 17,
      "source_field": "full_message",
      "target_field": "utm_setcookie",
      "title": "utm_setcookie"
    }
  ],
  "version": "1.2.2 (91c7822)"
}

Reply via email to