ELK Stack

Working solution

Prepare docker-compoase.yml

version: '3.7'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
    ports:
      - "9200:9200"
    volumes:
      - esdata:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:7.17.1
    container_name: logstash
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
    ports:
      - "5044:5044"
      - "9600:9600"

  kibana:
    image: docker.elastic.co/kibana/kibana:7.17.1
    container_name: kibana
    environment:
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    ports:
      - "5601:5601"

volumes:
  esdata:

Configure Logstash Pipeline

Create a logstash.conf file in the logstash/pipeline directory to configure how Logstash processes Nginx logs.

- docker-compose.yml
- logstash
  - pipeline
    - logstash.conf

logstash.conf

input {
  beats {
    port => 5044
  }
}

filter {
  grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
  }
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "nginx-logs-%{+YYYY.MM.dd}"
  }
}

Set Up Filebeat on Nginx Server

Filebeat is used to forward Nginx logs to Logstash. Create a filebeat.yml configuration file for Filebeat.

/etc/filebeat/filebeat.yml content:

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log

output.logstash:
  hosts: ["logstash:5044"]

Install Filebeat

wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt-get install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt-get update
sudo apt-get install filebeat

sudo systemctl start filebeat
sudo systemctl enable filebeat
sudo systemctl status filebeat

Start the ELK Stack

docker-compose up -d

Experimental (Include filebeat in docker-compose.yml)

Create a docker-compose.yml file

version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    ports:
      - "9200:9200"
    environment:
      - discovery.type=single-node

  logstash:
    image: docker.elastic.co/logstash/logstash:7.17.1
    volumes:
      - ./logstash/config:/usr/share/logstash/config
      - ./logstash/pipeline:/usr/share/logstash/pipeline
    ports:
      - "5044:5044"

  kibana:
    image: docker.elastic.co/kibana/kibana:7.17.1
    ports:
      - "5601:5601"
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200

  filebeat:
    image: docker.elastic.co/beats/filebeat:7.17.1
    volumes:
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/log/nginx:/var/log/nginx:ro  # Mount Nginx logs into Filebeat container
      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      - SETUP_KIBANA_HOST=kibana:5601
    depends_on:
      - elasticsearch
      - kibana

Create a filebeat.yml config

In the same directory as docker-compose.yml, create filebeat.yml

filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/nginx/access.log
      - /var/log/nginx/error.log

output.elasticsearch:
  hosts: ["${ELASTICSEARCH_HOSTS}"]

setup.kibana:
  host: "${SETUP_KIBANA_HOST}"

Comments