Created
July 24, 2012 22:27
-
-
Save KirstensAmazing/3173094 to your computer and use it in GitHub Desktop.
logstash GELF filtering
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
filter { | |
## This will pull out unnecessary and repeated celery information about it starting jobs. This goes first so that it doesn't get mutated later. | |
grep { | |
type => "celeryd" | |
match => ["@message", "^([D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?) [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} .*?$"] | |
negate => true | |
} | |
## This will capture the message in the format time: severity/process message . This is capturing the data and tagging it so that it can be exported to our GELF format later | |
grok { | |
match => ["@message", "\[%{DATESTAMP:timestamp}: %{DATA:severity}/%{DATA:process}\] %{DATA:message}$"] | |
keep_empty_captures => true | |
drop_if_match => false | |
} | |
## This sets the date into the appropriate gelf date format | |
date { | |
logdate => "yyyy-MM-dd HH:mm:ss,SSS" | |
} | |
## This will strip the timestamp out of the message so that it no longer appears in the GELF shortmessage. | |
mutate { | |
gsub => ["@message", "^\[[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3}: (?:.*?)/(?:.*?)\] ", "\1"] | |
} | |
} | |
output { | |
stdout { | |
debug => true | |
debug_format => "json" | |
} | |
gelf { | |
host => "logging1" | |
facility => "%{@type}/%{process}" | |
level => ["%{severity}", "INFO"] | |
port => 12205 | |
sender => "%{@source_host}" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment