Em Desenvolvimento
* Ajustar configuração do conf BEAT caso necessário, segue exemplos:
Baixar o pacote, caso usou a linha única de instalação dos serviços no inicio do tutorial pule esta parte.
# apt install nginx-extras openssl
Criar certificado e/ou definir segurança
# openssl req -x509 -days 3650 -nodes -newkey rsa:2048 -keyout /etc/ssl/private/<domain.com> ou <SERVER_IP_INT>.key -out /etc/ssl/certs/<domain.com> ou <SERVER_IP_INT>.pem.crt -subj "/C=BR/ST=PARANA/L=CURITIBA/O=EMPRESA/OU=Departamento TI/CN=<domain.com> ou <SERVER_IP_INT>/emailAddress=login@domain.com/"
# echo "admin:$(openssl passwd -apr1 SeuPassword)" | tee -a /etc/nginx/htpasswd.kibana
Altere o site default do nginx
# cp -a /etc/nginx/sites-available/default /etc/nginx/sites-available/default.00
# vi /etc/nginx/sites-available/default
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name <domain.com> ou <SERVER_IP_INT>;
return 301 https://$server_name$request_uri;
}
server {
listen 443 default_server ssl http2;
server_name <domain.com> ou <SERVER_IP_INT>;
ssl_certificate /etc/ssl/certs/<domain.com> ou <SERVER_IP_INT>.pem.crt;
ssl_certificate_key /etc/ssl/private/<domain.com> ou <SERVER_IP_INT>.key;
ssl_session_cache shared:SSL:10m;
auth_basic "Restrict Access";
auth_basic_user_file /etc/nginx/htpasswd.kibana;
location / {
proxy_pass http://<domain.com> ou <SERVER_IP_INT>:5601;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
location ~ /\.ht {
deny all;
}
}
Teste e reinicie caso as configurações estejam OK
# nginx -t
# systemctl restart nginx.service
Acesse http://<domain.com> ou <IP>
Quando for habilitado os módulos no filebeat o serviço precisa das funções do kibana e elasticsearch habilitado e logstash desabilitado.
# vi /etc/filebeat/filebeat.yml
#============================= Filebeat modules ===============================
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
Local de armazenamento dos módulos
# ls /etc/filebeat/modules.d/
Configurar o módulo haproxy:
# filebeat modules list
# filebeat modules enable haproxy
# vi /etc/filebeat/modules.d/haproxy.yml
- module: haproxy
log:
enabled: true
var.input: "file"
var.paths: ["/var/log/haproxy.log"]
# filebeat setup -e <- Aguardar finalizar o procedimento
# service filebeat restart
Instale o serviço haproxy conforme o tutorial e configure com as opções abaixo:
# vi /etc/haproxy/haproxy.cfg
global
log 127.0.0.1:9001 len 4096 local0
log 127.0.0.1:9001 len 4096 local1 debug <- emerg alert crit err warning notice info debug
defaults
log global
mode http <- Possível usar mode tcp
option httplog
option dontlog-normal
#option dontlognull
frontend www-http
bind *:80
default_backend b-www-http
backend b-www-http
server SRV 10.0.0.129:81 check
# service haproxy start
Configurar o módulo nginx:
# filebeat modules list
# filebeat modules enable nginx
# vi /etc/filebeat/modules.d/nginx.yml
- module: nginx
# Access logs
access:
enabled: true
var.paths: ["/var/log/nginx/access.log"]
var.convert_timezone: true
# Error logs
error:
enabled: true
var.paths: ["/var/log/nginx/error.log"]
var.convert_timezone: true
# filebeat setup -e <- Aguardar finalizar o procedimento
# service filebeat restart
Configurar os serviço Iptables
# vi /home/firewall
# chmod 755 /home/firewall
SERVER
IN
$IPT -A INPUT -p tcp -s $NETINT -m multiport --dport 22,80,443,5044,5601,9200,9600:9700 -i $IFINT -j ACCEPT
OUT
$IPT -A OUTPUT -p tcp -d $ANY -m multiport --dport 80,443 -o $IFINT -j ACCEPT
$IPT -A OUTPUT -p udp -d $ANY -m multiport --dport 53,123 -o $IFINT -j ACCEPT
CLIENTE
IN
iifname $IFINT ip saddr $NETINT tcp dport {ssh} ct state {new,established} counter accept # SERVICOS INT
iifname $IFEXT ip saddr $ANY tcp dport {http} ct state {new,established} counter accept # SERVICOS EXT
OUT
oifname $IFEXT tcp dport {http,https} counter accept # HTTP E HTTPS
oifname $IFINT tcp dport {5044,5601,9200,9600-9700} counter accept # ELK STACK
http://<ipserverelk>:9200/filebeat-*/_search?pretty
# /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -e -v -d "*"
Instalar e configurar o rsyslog
global
#log 127.0.0.1:514 len 4096 local1 <- Porta 514 utiliza serviço rsyslog
# apt install rsyslog
# vi /etc/rsyslog.conf
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support
$ModLoad imudp
$UDPServerAddress 0.0.0.0 # listen on the localhost , protocol UDP
$UDPServerRun 514 # listen on port 514, protocol UDP
$KLogPermitNonKernelFacility on
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$RepeatedMsgReduction on
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog
$WorkDirectory /var/spool/rsyslog
# the syslog template,
$template HAProxyFormat,"[HAPROXY_TOKEN] <%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [type=haproxy] %msg%\n"
# Send messages to Logz over TCP using the template.
*.* @@<SERVER_IP_INT>:5000;HAProxyFormat
# service rsyslog restart
Ver haproxy plugin do kibana ou elasticsearch
Instalar e/ou atualizar plugins
# /usr/share/elasticsearch/bin/elasticsearch-plugin update
# /usr/share/elasticsearch/bin/elasticsearch-plugin update
[2019-07-03T08:52:57,764][INFO ][o.e.p.PluginsService ] [node-1] no plugins loaded
# /usr/share/logstash/bin/elasticsearch-plugin install discovery-ec2
*
Instalar e/ou atualizar plugins
$ /usr/share/kibana/bin/kibana-plugin update
OU
# /usr/share/kibana/bin/kibana-plugin update --allow-root
Instalar e/ou atualizar plugins
# /usr/share/logstash/bin/logstash-plugin list <- Lista os plugins instalados ???????
# /usr/share/logstash/bin/logstash-plugin update
# /usr/share/logstash/bin/logstash-plugin install logstash-input-beats
Configurar certificado <- Necessário o pacote X-Pack
# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
# /usr/share/elasticsearch/bin/elasticsearch-certutil cert
Please enter the desired output file [elastic-certificates.p12]: /etc/elasticsearch/elastic-certificates.p12
Enter password for elastic-certificates.p12 : <PRESS_ENTER> ou <INSIRA_SENHA>
# vi /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12
# service elasticsearch restart
# /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto
y
# vi kibana.yml
elasticsearch.username: "kibana"
elasticsearch.password: "<SENHA COMANDO ACIMA>"
# service kibana restart
Video com dicas de possibilidades de uso - https://www.elastic.co/pt/webinars/canvas-a-single-and-stunning-pane-of-glass-for-logs-metrics-and-apm?baymax=rtp&storm=blog-sidebar&elektra=canvas-int-ess-trial&iesrc=ctr
inicia 21:30 / 22:50
* A cada 365 dias sera necessário renovar os certificados
NGINX
# openssl req -x509 -days 3650 -nodes -newkey rsa:2048 -keyout /etc/ssl/private/<domain.com> ou <SERVER_IP_INT>.key -out /etc/ssl/certs/<domain.com> ou <SERVER_IP_INT>.pem.crt -subj "/C=BR/ST=PARANA/L=CURITIBA/O=EMPRESA/OU=Departamento TI/CN=<domain.com> ou <SERVER_IP_INT>/emailAddress=login@domain.com/"
LOGSTASH
# openssl req -x509 -days 3650 -nodes -newkey rsa:2048 -batch -keyout /etc/logstash/logstash.key -out /etc/logstash/logstash.pem.crt -config /etc/logstash/logstash-openssl.cnf
# openssl pkcs8 -in /etc/logstash/logstash.key -topk8 -nocrypt -out /etc/logstash/logstash.pkcs8.key
# chmod 644 /etc/logstash/logstash.pkcs8.key
KIBANA
# openssl req -x509 -days 365 -nodes -newkey rsa:2048 -keyout /etc/ssl/private/kibana.key -out /etc/ssl/certs/kibana.pem.crt -subj "/C=BR/ST=PARANA/L=CURITIBA/O=EMPRESA/OU=Departamento TI/CN=<domain.com> ou <SERVER_IP_INT>/emailAddress=login@domain.com/"
ELASTICSEARCH
# openssl req -x509 -days 365 -nodes -newkey rsa:2048 -keyout /etc/ssl/private/elasticsearch.key -out /etc/ssl/certs/elasticsearch.pem.crt -subj "/C=BR/ST=PARANA/L=CURITIBA/O=EMPRESA/OU=Departamento TI/CN=<domain.com> ou <SERVER_IP_INT>/emailAddress=login@domain.com/"
Dica: Criar script e agendar no cron
Baixar o pacote
# apt install apm-server
Configuração
# vi /etc/apm-server/apm-server.yml
https://logz.io/learn/complete-guide-elk-stack
https://www.youtube.com/watch?v=LapNa2l-7VA
https://www.rosehosting.com/blog/how-to-install-the-elk-stack-on-debian-9/
https://www.itzgeek.com/how-tos/linux/debian/how-to-setup-elk-stack-on-debian-9-debian-8.html
https://logz.io/blog/monitor-haproxy-elk-stack/
https://logstail.com/blog/monitor-haproxy-with-elk-stack/
https://www.youtube.com/watch?v=tcB3Gc5Oycg
https://www.youtube.com/watch?v=P0phFCuW79o
https://www.youtube.com/watch?v=MRMgd6E9AXE
https://www.youtube.com/watch?v=LapNa2l-7VA - 1:08:40
https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-haproxy.html
Conf para o logstash
# vi /etc/logstash/conf.d/02-syslog.conf
input {
tcp {
type => "syslog"
port => 5140
}
}
input {
udp {
type => "syslog"
port => 5140
}
}
Baixe o arquivo Filebeat Windows zip da página.
Extraia em C:\Program Files e renomei para Filebeat.
Abra o PowerShell como administrador e execute:
PS > cd 'C:\Filebeat'
Ajuste o arquivo de configuração
#========= Filebeat inputs =================
filebeat.inputs:
- type: log
enabled: true
paths:
- C:\filebeat\logs\*
# =================================== Kibana ===================================
#setup.kibana:
#================= Outputs ====================
# Decidir qual utilizar, Elasticsearch ou logstash
# ---------------------------- Elasticsearch Output ----------------------------
#output.elasticsearch:
# hosts: ["<IP_SERVER_ELK>:9200"]
#-------------------- Logstash output ------------------------
output.logstash:
hosts: ["<IP_SERVER_ELK>:5044"]
# List of root certificates for HTTPS server verifications
# Copiar a chave gerada no servidor do logstash e disponibilizar com permissao 644
ssl.certificate_authorities: ["/etc/ssl/certs/logstash.pem.crt"]
Testar a configuração
PS C:\Filebeat> .\filebeat.exe test config -c .\filebeat.yml -e
Configurar o serviço no Start-Service
PS C:\Filebeat> .\install-service-filebeat.ps1
Status Name DisplayName
------ ---- -----------
Stopped filebeat filebeat
Iniciar o serviço
PS C:\filebeat> Start-Service filebeat
Modulos especificos (Em desenvolvimento)
PS > .\filebeat.exe modules list
PS > .\filebeat.exe modules enable system microsoft mssql o365 system
# vi /etc/elasticsearch/elasticsearch.yml
#-------------------------- Cluster --------------------------------------
cluster.namer: my-application <- Caso deseje ajustar o nome para o cluster altere aqui
# ------------------------- Node -----------------------------------------
node.name: node-1 <- Opcional
# ----------------------------------- Paths ------------------------------------
path.data: /path/SYSLOG/elasticsearch <- Manter default para armazenar local
path.logs: /path/SYSLOG/elasticsearch <- Manter default para armazenar local
#-------------------------- Network -------------------------------------
network.host: localhost <- Manter esta linha comentada para funcionar como localhost
network.publish_host: <IP_SERVER>
http.port: 9200
####transport.tcp.port: 9300 <- Rever documentação
#------------------------- Discovery ----------------------------------
cluster.initial_master_nodes: ["<IP_SERVE>"]