Logstash

logstash-syslog.sh

#!/usr/bin/env bash

echo ">>> Setup logstash"

sudo apt install logstash
sudo systemctl enable logstash
sudo systemctl start logstash

cat <<EOF | sudo tee /etc/rsyslog.d/01-json-template.conf
template(name="json-template"
  type="list") {
    constant(value="{")
      constant(value="\"@timestamp\":\"")     property(name="timereported" dateFormat="rfc3339")
      constant(value="\",\"@version\":\"1")
      constant(value="\",\"message\":\"")     property(name="msg" format="json" position.from="2")
      constant(value="\",\"sysloghost\":\"")  property(name="hostname")
      constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")
      constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")
      constant(value="\",\"programname\":\"") property(name="programname")
      constant(value="\",\"procid\":\"")      property(name="procid")
    constant(value="\"}\n")
}
EOF

cat <<EOF | sudo tee /etc/logstash/conf.d/logstash.conf
input {
  udp {
    host => "127.0.0.1"
    port => 10514
    codec => "json"
    type => "rsyslog"
  }
}

# The Filter pipeline stays empty here, no formatting is done.
filter {
  if [message] =~ "\A\{.+\}\z" {
    json {
      source => "message"
    }
  }
}

# Every single log will be forwarded to ElasticSearch. If you are using another port, you should specify it here.
output {
  if [type] == "rsyslog" {
    elasticsearch {
      hosts => [ "127.0.0.1:9200" ]
    }
  }
}
EOF

cat <<EOF | sudo tee /etc/rsyslog.d/70-output.conf
# This line sends all lines to defined IP address at port 10514
# using the json-template format.

*.*                         @127.0.0.1:10514;json-template
EOF

sudo service rsyslog restart
sudo service logstash restart