Logstash Nginx 日志解析管道

目标

使用 Logstash 解析 Nginx Access Log，提取结构化字段，写入 Elasticsearch。

完整配置

logstash-nginx.conf

input {
  # 方案A：读取文件
  file {
    path => "/var/log/nginx/access*.log"
    start_position => "beginning"
    sincedb_path => "/var/lib/logstash/nginx_sincedb"
    codec => plain
  }

  # 方案B：通过 Beats 接收（推荐生产用）
  beats {
    port => 5044
  }
}

filter {
  # 标准 Nginx 日志格式解析
  grok {
    match => {
      "message" => '%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} (?:%{NUMBER:body_bytes_sent}|-) "(?:%{DATA:http_referer}|-)" "%{DATA:http_user_agent}"'
    }
  }

  # 时间字段转换
  date {
    match => ["timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
    target => "@timestamp"
  }

  # 地理位置解析（基于 IP）
  geoip {
    source => "remote_addr"
    target => "geoip"
  }

  # 用户代理解析
  useragent {
    source => "http_user_agent"
    target => "ua"
  }

  # 添加字段
  mutate {
    convert => {
      "body_bytes_sent" => "integer"
      "status" => "integer"
    }
    add_field => {
      "environment" => "production"
    }
    remove_field => ["message", "@version"]
  }
}

output {
  # 写入 Elasticsearch
  elasticsearch {
    hosts => ["https://elasticsearch:9200"]
    index => "nginx-access-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "${ES_PASSWORD}"
    ssl => true
    ssl_certificate_verification => false
  }

  # 调试用：控制台输出（仅开发环境）
  stdout {
    codec => rubydebug
  }
}

运行步骤

# 1. 验证配置
logstash -f logstash-nginx.conf --config.test_and_exit

# 2. 运行
logstash -f logstash-nginx.conf

# 3. Docker 方式
docker run -d --name logstash-nginx \
  -v $(pwd)/logstash-nginx.conf:/usr/share/logstash/pipeline/logstash.conf:ro \
  -v /var/log/nginx:/var/log/nginx:ro \
  docker.elastic.co/logstash/logstash:8.12.0

预期输出

解析后的日志在 Elasticsearch 中可查询结构化字段（status、method、geoip.country_name 等），并在 Kibana 中可视化。