I have a simple express app with Nginx and I use Filebeat with ELK stack. Filebeat takes in charge of streaming log file from Nginx to Logstash then processing it and visualize to Kibana. This pipeline works fine. However, I wanted to check how can I incorporate Packetbeat to this pipeline to check the different possibilities that gives to me.
My docker-compose.yml
is the following:
version: '3.2'
services:
elasticsearch:
build:
context: elasticsearch/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./elasticsearch/config/elasticsearch.yml
target: /usr/share/elasticsearch/config/elasticsearch.yml
read_only: true
- type: volume
source: elasticsearch
target: /usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
ELASTIC_PASSWORD: changeme
networks:
- elk
logstash:
build:
context: logstash/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./logstash/config/logstash.yml
target: /usr/share/logstash/config/logstash.yml
read_only: true
- type: bind
source: ./logstash/pipeline
target: /usr/share/logstash/pipeline
read_only: true
ports:
- "5000:5000"
- "9600:9600"
expose:
- "5044"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elk
depends_on:
- elasticsearch
kibana:
build:
context: kibana/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./kibana/config/kibana.yml
target: /usr/share/kibana/config/kibana.yml
read_only: true
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
app:
build : ./app
volumes:
- ./app/:/usr/src/app
- /usr/src/app/node_modules/ # make node_module empty in container
command: npm start
ports:
- "3000:3000"
networks:
- elk
nginx:
build: ./nginx
volumes:
- ./nginx/config:/etc/nginx/conf.d
- ./nginx/log:/var/log/nginx
ports:
- "80:80"
- "443:443"
links:
- app:app
depends_on:
- app
networks:
- elk
filebeat:
build: ./filebeat
entrypoint: "filebeat -e -strict.perms=false"
volumes:
- ./filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
- ./nginx/log:/var/log/nginx
networks:
- elk
depends_on:
- app
- nginx
- logstash
- elasticsearch
- kibana
links:
- logstash
packetbeat:
build:
context: packetbeat/
args:
ELK_VERSION: $ELK_VERSION
command: -e --strict.perms=false
restart: always
volumes:
- type: bind
source: ./packetbeat/config/packetbeat.yml
target: /usr/share/packetbeat/config/packetbeat.yml
read_only: true
user: packetbeat
cap_add:
- NET_RAW
- NET_ADMIN
network_mode: host
depends_on:
- elasticsearch
- kibana
- app
- nginx
- logstash
networks:
elk:
driver: bridge
volumes:
elasticsearch:
The packetbeat.yml
for configuration is the following:
packetbeat.interfaces.device: any
packetbeat.flows:
timeout: 30s
period: 10s
packetbeat.protocols.dns:
ports: [53]
include_authorities: true
include_additionals: true
packetbeat.protocols.http:
ports: [80, 5601, 9200, 8080, 8081, 5000, 8002]
packetbeat.protocols.memcache:
ports: [11211]
packetbeat.protocols.mysql:
ports: [3306]
packetbeat.protocols.pgsql:
ports: [5432]
packetbeat.protocols.redis:
ports: [6379]
packetbeat.protocols.thrift:
ports: [9090]
packetbeat.protocols.mongodb:
ports: [27017]
packetbeat.protocols.cassandra:
ports: [9042]
processors:
- add_cloud_metadata: ~
setup.kibana:
host: "kibana:5601/kibana"
username: "elastic"
password: "changeme"
file:
path: "./packetbeat"
filename: packetbeat
rotate_every_kb: 10000
number_of_files: 7
output.elasticsearch:
hosts: ["http://elasticsearch:9200"]
username: "elastic"
password: "changeme"
logging:
files:
rotateeverybytes: 10485760
The error that I am getting is the following:
022-05-17T07:27:04.701Z ERROR pipeline/output.go:100 Failed to connect to backoff(elasticsearch(http://elasticsearch:9200)): Get http://elasticsearch:9200: lookup elasticsearch on 192.168.65.5:53: no such host
Any idea of where is the mistake?