Sample Vector configuration for nginx

nginx.toml

# be careful, you will gets tons of logs 
# you may want to skip access logs or use sampling only here
[sources.nginx_access_logs]
  type         = "file"
  include      = ["/var/log/sites/*/*/*/*/nginx/access.log"]    # supports globbing
  ignore_older = 86400                         # 1 day

# I'm configuring my nginx to use json logs, if you use another logging format you need to change this
[transforms.nginx_access_logs_json]
  type         = "json_parser"
  inputs       = ["nginx_access_logs"]

# Here I want to add an additionaly type telling the type of log
[transforms.nginx_access_logs_fields]
  type         = "add_fields"
  inputs       = ["nginx_access_logs_json"]
  overwrite    = false
  fields.type  = "nginx_access"

# sampling
[transforms.nginx_access_sampler]
  inputs       = ["nginx_access_logs_fields"]
  type         = "sampler"
  rate         = 50                            # only keep 50%

[sources.nginx_error_logs]
  type         = "file"
  include      = ["/var/log/sites/*/*/*/*/nginx/error.log"]    # supports globbing
  ignore_older = 86400

# nginx is weird, it does not supports json logs for error
[transforms.nginx_error_logs_parsed]
  type         = "grok_parser"
  inputs       = ["nginx_error_logs"]
  pattern      = '(?<timestamp>%{YEAR}[./]%{MONTHNUM}[./]%{MONTHDAY} %{TIME}) \[%{LOGLEVEL:level}\] %{POSINT:pid}#%{NUMBER:threadid}\: \*%{NUMBER:connectionid} %{GREEDYDATA:message}, client: %{IP:client}, server: %{GREEDYDATA:server}, request: "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion}))"(, upstream: "%{GREEDYDATA:upstream}")?, host: "%{DATA:host}"(, referrer: "%{GREEDYDATA:referrer}")?'
  types.timestamp = "timestamp|%s" # timestamp conversion

[transforms.nginx_error_logs_fields]
  type         = "add_fields"
  inputs       = ["nginx_error_logs_parsed"]
  overwrite    = false
  fields.type  = "nginx_error"

[sinks.my_log_service]
  # General
  type = "http" # required
  inputs = ["nginx_access_logs_fields", "nginx_error_logs_fields"] # required

  compression = "none" # optional, default
#  healthcheck = true # optional, default
 
   uri = "your uri here"

  # Batch
  batch.max_bytes = 1049000 # optional, default, bytes
  batch.max_events = 1000 # optional, no default, events
  batch.timeout_secs = 1 # optional, default, seconds

  # Buffer
  buffer.max_events = 500 # optional, default, events, relevant when type = "memory"
  buffer.type = "memory" # optional, default

  # Encoding
  encoding.codec = "json" # required
  encoding.timestamp_format = "unix"

  #  Header
  headers.Authorization = "someauththorizationtokenhere"

  # Request
  request.in_flight_limit = 10 # optional, default, requests
  request.rate_limit_duration_secs = 1 # optional, default, seconds
  request.rate_limit_num = 1000 # optional, default
  request.timeout_secs = 30 # optional, default, seconds