svk
/
elasticsearch
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial release

master
Sergey Kalinin 2021-01-15 18:19:45 +03:00
commit c3c29925cf
26 changed files with 2028 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md 100644
View File

@ -0,0 +1,11 @@
# Что тут
Тут лежат всё что связано с elasticsearch
# Состав
- cerebro - это ВЭБ-сервис для управления ES. В каталоге скрипты для запуска cerebro (https://github.com/lmenezes/cerebro) в docker
- curator - всё для сборки и запуска сервиса в docker. Curator это сервис управления индексами elasticsearch https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html
- pipelines - шаблоны для разборки различных данных в elasticsearch ingestnode.
- scripts - разные вспомогательные скрипты
- zabbix_template - шаблон для мониторинга ES-кластера

4
cerebro/.env 100644
View File

@ -0,0 +1,4 @@
NGINX_HOST=elk.example.com
CEREBRO_PASSWORD=some_password
CEREBRO_USER=admin

21
cerebro/Dockerfile 100644
View File

@ -0,0 +1,21 @@
FROM openjdk:11-jre-slim
ENV CEREBRO_VERSION 0.9.2
RUN apt-get update \
&& apt-get install -y wget \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /opt/cerebro/logs \
&& wget -qO- https://github.com/lmenezes/cerebro/releases/download/v${CEREBRO_VERSION}/cerebro-${CEREBRO_VERSION}.tgz \
| tar xzv --strip-components 1 -C /opt/cerebro \
&& sed -i '/<appender-ref ref="FILE"\/>/d' /opt/cerebro/conf/logback.xml \
&& addgroup -gid 1000 cerebro \
&& adduser -gid 1000 -uid 1000 cerebro \
&& chown -R cerebro:cerebro /opt/cerebro
RUN echo "play.ws.ssl.loose.acceptAnyCertificate = true" >> /opt/cerebro/conf/reference.conf
WORKDIR /opt/cerebro
USER cerebro
ENTRYPOINT [ "/opt/cerebro/bin/cerebro" ]

37
cerebro/docker-compose.yml 100644
View File

@ -0,0 +1,37 @@
version: '3'
services:
cerebro:
image: cerebro:latest
container_name: cerebro
build:
context: ./
restart: always
# ports:
# - 5555:5000
# expose:
# - "5555"
environment:
- AUTH_TYPE=basic
- BASIC_AUTH_USER=${CEREBRO_USER:-admin}
- BASIC_AUTH_PWD=${CEREBRO_PASSWORD:-admin}
networks:
- odfe-net
depends_on:
- nginx
nginx:
image: nginx
container_name: nginx
env_file:
- .env
restart: always
ports:
- 443:443
environment:
- NGINX_HOST=${NGINX_HOST}
volumes:
- ./nginx/templates:/etc/nginx/templates
- ./ssl/node.pem:/etc/nginx/certs/nginx-selfsigned.pem:ro
- ./ssl/node.key:/etc/nginx/certs/nginx-selfsigned.key:ro
networks:
- odfe-net

15
cerebro/nginx/templates/elastic.conf.template 100644
View File

@ -0,0 +1,15 @@
server {
listen 443 ssl;
server_name ${NGINX_HOST};
client_max_body_size 100M;
ssl_certificate /etc/nginx/certs/nginx-selfsigned.pem;
ssl_certificate_key /etc/nginx/certs/nginx-selfsigned.key;
location / {
proxy_pass http://cerebro:9000;
# proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}

2
curator/.env 100644
View File

@ -0,0 +1,2 @@
ELASTIC_HOST=elk.example.com
CURATOR=admin:some_password

18
curator/Dockerfile 100644
View File

@ -0,0 +1,18 @@
FROM debian:buster-slim
RUN apt update && apt install -y wget gnupg2
RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -
RUN echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian9 stable main" > /etc/apt/sources.list.d/curator.list
RUN apt update -y && apt install -y cron elasticsearch-curator tini
ENV CURATOR_VERSION=5.8.1
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
COPY entrypoint.sh /
RUN ["chmod", "+x", "/entrypoint.sh"]
WORKDIR /usr/share/curator
ENTRYPOINT ["/entrypoint.sh"]

18
curator/config/action_delete_alerts.yml 100644
View File

@ -0,0 +1,18 @@
actions:
5:
action: delete_indices
description: >-
Delete indices. Find which to delete by first limiting the list to logstash-
prefixed indices.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: .opendistro-alerting-
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 10

17
curator/config/action_delete_filebeat.yml 100644
View File

@ -0,0 +1,17 @@
actions:
4:
action: delete_indices
description: >-
Delete indices older than
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: filebeat-
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 10

18
curator/config/action_delete_logstash.yml 100644
View File

@ -0,0 +1,18 @@
actions:
1:
action: delete_indices
description: >-
Delete indices. Find which to delete by first limiting the list to logstash-
prefixed indices.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: logstash-
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 2

17
curator/config/action_delete_maillog.yml 100644
View File

@ -0,0 +1,17 @@
actions:
7:
action: delete_indices
description: >-
Delete indices older than
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: maillog_filebeat-
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 30

18
curator/config/action_delete_winlogbeat.yml 100644
View File

@ -0,0 +1,18 @@
actions:
3:
action: delete_indices
description: >-
Delete indices. Find which to delete by first limiting the list to logstash-
prefixed indices.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: winlogbeat-
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 10

24
curator/config/curator.yml 100644
View File

@ -0,0 +1,24 @@
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
client:
hosts:
- ${ELASTICSEARCH_HOST}
port: 9200
url_prefix:
use_ssl: True
certificate: /etc/cert.pem
client_cert:
client_key:
aws_key:
aws_secret_key:
aws_region:
ssl_no_validate: True
http_auth: ${CURATOR}
timeout: 30
master_only: False
logging:
loglevel: INFO
logfile:
logformat: default

18
curator/docker-compose.yml 100644
View File

@ -0,0 +1,18 @@
version: '3'
services:
curator:
build:
context: curator/
image: curator:latest
restart: always
container_name: curator
env_file:
- .env
environment:
ELASTICSEARCH_HOST: ${ELASTIC_HOST}
CRON: "0 0 * * *"
volumes:
- ./curator/config:/usr/share/curator/config
- ./ssl/MyRootCA.pem:/etc/cert.pem:ro
networks:
- odfe-net

12
curator/entrypoint.sh 100755
View File

@ -0,0 +1,12 @@
#!/bin/sh
#shopt -s nullglob
FILES=/usr/share/curator/config/action_*
for f in $FILES
do
echo "$CRON /usr/bin/curator --config /usr/share/curator/config/curator.yml $f" >> /etc/crontab
done
# https://github.com/krallin/tini/blob/master/README.md#subreaping
tini -s -- cron -f -L 8

14
pipelines/README.md 100644
View File

@ -0,0 +1,14 @@
# Шаблоны для создания pipelines в ingestnode ES.
- amavis.json - обработка логов amavis
- cdr.json - разбор логов от АТС (avaya, asterisk)
- fail2ban.json - разбор лога F2B
- mailboxlog.json - обработка лога mailboxlog от Zimbra
- maillog.json - шаблон для обработки логов (maillog, fail2ban.log, zibmra/audit.log, zimbra/nginx.access.log)
- maillog_with_geoip.json - шаблон pipeline для получения GEO данных, на основе полей создаваемых предыдущим (maillog) (используется последовательный запуск)
- zaimbralog.json - парсинг почтового лога Zimbra
# Внесение изменений в elasticsearch
```curl -X POST "elk.example.com:9200"/_ingest/pipeline/maillog" -H 'Content-Type: application/json' -d '@maillog.json'```
```curl -X POST "elk.example.com:9200"/_ingest/pipeline/maillog_with_geoip" -H 'Content-Type: application/json' -d '@maillog_with_geoip.json'```

188
pipelines/amavis.json 100644
View File

@ -0,0 +1,188 @@
{
"processors": [
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{WORD:mail.command}: %{WORD:mail.amavis_result} %{DATA:mail.reason}, From: <%{EMAIL:mail.from}>"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result}, <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ %{DATA:mail.amavis_spam_tag}, score=%{DATA:mail.amavis_spam_score} required=%{DATA} tests=\\[%{GREEDYDATA:mail.amavis_spam_result}\\]"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} %{DATA:mail.amavis_result} from <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ %{DATA:mail.reason}: queued as %{QUEUEID:mail.qid}"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, %{DATA:mail.reason} \\[%{IP:mail.host}\\]:%{PORT:mail.port} <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, queued_as: %{QUEUEID:mail.amavis_qid}, dkim_sd=%{DATA:mail.dkim}, %{POSINT:mail.amavis_delay} ms"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} :%{PORT:mail.port} %{DATA:mail.amavis_file}: <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}> (SIZE=%{POSINT:mail.size} |)(BODY=%{WORD} |)%{DATA:mail.amavis_result}: from %{HOSTNAME:mail.remote_host} \\(\\[%{IP:mail.remote_ip}\\]\\) by %{DATA} \\(%{HOSTNAME:mail.host} \\[%{IP:mail.ip}\\]\\) \\(%{DATA}\\) %{DATA:mail.reason}; %{GREEDYDATA:mail.amavis_datetime}"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result}: %{DATA:mail.amavis_qid} (%{DATA:mail.reason} |)\\[%{IP:mail.remote_ip}\\] <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{WORD:mail.amavis_qid} %{DATA:mail.amavis_result} from <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, %{DATA} from %{WORD}\\(%{WORD}:\\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port}\\): %{DATA:mail.reason}: queued as %{WORD:mail.qid}"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, \\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port} \\[%{IP}\\] <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, (queued_as: %{QUEUEID:mail.amavis_qid}, |)%{POSINT:mail.amavis_delay} ms"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, %{DATA:mail.reason} \\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port} <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, queued_as: %{QUEUEID:mail.amavis_qid}, %{POSINT:mail.amavis_delay} ms"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) size: %{POSINT:mail.size}, %{WORD:mail.amavis_result} \\[total %{POSINT:mail.amavis_check.total} ms, cpu %{POSINT:mail.amavis_check.cpu} ms((, AM-cpu %{POSINT} ms)|)((, SA-cpu %{POSINT} ms)|)\\] - SMTP greeting: %{BASE10NUM:mail.amavis_check.smtp_greeting} \\(%{PORT:mail.amavis_check.smtp_greeting_percent}\\%\\)%{WORD}, SMTP EHLO: %{BASE10NUM:mail.amavis_check.smtp_ehlo} \\(%{PORT:mail.amavis_check.smtp_ehlo_percent}\\%\\)%{WORD}, SMTP pre-MAIL: %{BASE10NUM:mail.amavis_check.smtp_pre_mail} \\(%{PORT:mail.amavis_check.smtp_pre_mail_percent}\\%\\)%{WORD}((, lookup_ldap: %{BASE10NUM:mail.amavis_check.lookup_ldap} \\(%{PORT:mail.amavis_check.lookup_ldap_percent}\\%\\)%{WORD})+), SMTP pre-DATA-flush: %{BASE10NUM:mail.amavis_check.smtp_pre_data_flush} \\(%{PORT:mail.amavis_check.smtp_pre_data_flush_percent}\\%\\)%{WORD}, SMTP DATA: %{BASE10NUM:mail.amavis_check.smtp_data} \\(%{PORT:mail.amavis_check.smtp_data_percent}\\%\\)%{WORD}, %{DATA}mime_decode: %{BASE10NUM:mail.amavis_check.mime_decode} \\(%{PORT:mail.amavis_check.mime_decode_percent}\\%\\)%{WORD}, %{DATA}SMTP pre-response: %{BASE10NUM:mail.amavis_check.smtp_pre_response} \\(%{PORT:mail.amavis_check.smtp_pre_response_percent}\\%\\)%{WORD}, SMTP response: %{BASE10NUM:mail.amavis_check.smtp_response} \\(%{PORT:mail.amavis_check.smtp_response_percent}\\%\\)%{WORD}"
],
"pattern_definitions" : {
"PORT" : "(?:[0-9]+)",
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} Content-Type: %{DATA:mail.content_type}(((, size: %{DATA:mail.size} B, name:)|)(( %{GREEDYDATA:mail.file_name})|)|)$"
],
"pattern_definitions" : {
"PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]"
},
"ignore_failure" : true
}
}
]
}

118
pipelines/cdr.json 100644
View File

@ -0,0 +1,118 @@
{
"processors" : [
{
"set": {
"field": "cdr.hour",
"value": "0"
}
},
{
"set": {
"field": "cdr.minute",
"value": "0"
}
},
{
"set": {
"field": "cdr.second",
"value": "0"
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:cdr.call_start},%{TIME_CUSTOM:cdr.call_duration_time},%{INT:cdr.ring_duration},%{DATA:cdr.caller_number},%{WORD:cdr.direction},%{DATA:cdr.called_number},%{DATA:cdr.dialed_number},%{DATA:cdr.account},%{INT:cdr.is_internal},%{INT:cdr.call_id},%{DATA:cdr.continuation},%{DATA:cdr.party1device},%{DATA:cdr.party1name},%{DATA:cdr.party2device},%{DATA:cdr.party2name},%{DATA:cdr.holdtime},%{DATA:cdr.park_time},%{DATA:cdr.field_1},%{DATA:cdr.field_2},%{DATA:cdr.field_3},%{DATA:cdr.field_4},%{DATA:cdr.field_5},%{DATA:cdr.field_6},%{DATA:cdr.field_7},%{DATA:cdr.field_8},%{DATA:cdr.field_9},%{DATA:cdr.field_10},%{DATA:cdr.field_11},%{DATA:cdr.field_12},%{DATA:cdr.field_13},%{HOST:cdr.hostname},%{DATA:cdr.field_14},%{HOST:cdr.field_15},%{DATA:cdr.field_16},%{LOGDATE}"
],
"pattern_definitions" : {
"TIME_CUSTOM" : "%{HOUR_CUSTOM:cdr.hour}:%{MINUTE:cdr.minute}(?::%{SECOND_CUSTOM:cdr.second})",
"HOUR_CUSTOM" : "(?!<[0-9])%{HOUR}",
"SECOND_CUSTOM" : "(?:(?:[0-5]?[0-9]|60))",
"HOST": "%{HOSTNAME}|%{IP}",
"LOGDATE" : "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"(<%{DATA}>|)%{LOGDATE:cdr.log_date} %{HOST:cdr.hostname} cdr-master: \"%{ACCOUNT:cdr.account}\",\"%{POSINT:cdr.caller_number}\",\"%{WORD:cdr.called_number}\",%{DATA:cdr.direction},\"%{CALL_DATE:cdr.call_start}\",\"(%{DATA}|)\",\"%{CALL_DATE:cdr.call_end}\",\"%{INT:cdr.call_duration}\",\"%{DATA}\",\"%{DATA:cdr.call_status}\""
],
"pattern_definitions" : {
"TIME_CUSTOM": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND_CUSTOM})",
"SECOND_CUSTOM": "(?:(?:[0-5]?[0-9]|60))",
"LOGDATE": "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"HOST": "%{HOSTNAME}|%{IP}",
"CALL_DATE": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}(\\s+)%{TIME}",
"ACCOUNT": "\"(%{DATA}|)\" <%{WORD}>"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"(<%{DATA}>|)%{LOGDATE:cdr.log_date} %{HOST:cdr.hostname} cdr-master: \"%{INT:cdr.call_duration}\",\"%{WORD:cdr.billsec}\",\"%{DATA:cdr.caller_number}\",\"%{DATA:cdr.channel}\",\"%{WORD:cdr.called_number}\",\"%{WORD:cdr.dialed_number}\",\"%{DATA}\",\"%{DATA:cdr.dst_channel}\",\"%{DATA:cdr.call_id}\""
],
"pattern_definitions" : {
"TIME_CUSTOM": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND_CUSTOM})",
"SECOND_CUSTOM": "(?:(?:[0-5]?[0-9]|60))",
"LOGDATE": "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"HOST": "%{HOSTNAME}|%{IP}",
"ACCOUNT": "((\"(%{DATA}|)\" <%{WORD}>)|(%{WORD}))"
},
"ignore_failure" : true
}
},
{
"convert" : {
"field" : "cdr.hour",
"type": "integer"
}
},
{
"convert" : {
"field" : "cdr.minute",
"type": "integer"
}
},
{
"convert" : {
"field" : "cdr.second",
"type": "integer"
}
},
{
"script": {
"source": "ctx.cdr.call_duration = ctx.cdr.hour * 3600 + ctx.cdr.minute * 60 + ctx.cdr.second"
}
},
{
"remove" : {
"field" : "cdr.hour"
}
},
{
"remove" : {
"field" : "cdr.minute"
}
},
{
"remove" : {
"field" : "cdr.second"
}
}
],
"on_failure" : [
{
"set" : {
"field" : "error",
"value" : "{{_ingest.on_failure_message}}"
}
}
]
}

21
pipelines/fail2ban.json 100644
View File

@ -0,0 +1,21 @@
{
"processors": [
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:f2b.log_datetime} %{F2BSERVICE} (\\s.*)\\[%{POSINT:f2b.pid}\\]: %{SEVERITY:f2b.severity} (\\s.*)\\[%{DATA:f2b.iptables_chain}\\] %{F2BMESSAGE}"
],
"pattern_definitions" : {
"REASON" : "(?:.+)",
"SEVERITY" : "(?:.+)",
"F2BSERVICE" : "%{WORD}.%{WORD:f2b.service}",
"F2BACTION" : "%{GREEDYDATA:f2b.action}",
"F2BMESSAGE" : "(%{F2BACTION} %{IP:f2b.remote_ip})|(%{IP:f2b.remote_ip} %{F2BACTION})",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}"
},
"ignore_failure" : true
}
}
]
}

189
pipelines/mailboxlog.json 100644
View File

@ -0,0 +1,189 @@
{
"processors" : [
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] mailop - (%{MOVINGMESSAGE}|%{ADDINGMESSAGE}|%{DELETEMESSAGE})"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}(|-%{POSINT:mail.pid})(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(||ip=%{IP:mail.ip};)",
"ADDINGMESSAGE" : "%{DATA:mail.command}: id=%{NONNEGINT:mail.id}, Message-ID=<%{DATA:mail.message_id}>, parentId=%{DATA}, folderId=%{NONNEGINT:mail.folder_id}, folderName=%{DATA:mail.folder_name} acct=%{NUMBER:mail.acct}.",
"MOVINGMESSAGE" : "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\) to Folder %{DATA:mail.folder_name} \\(id=%{NONNEGINT:mail.folder_id}\\)",
"DELETEMESSAGE" : "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\).",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] nginxlookup - %{DATA:mail.reason}:%{GREEDYDATA:mail.client_name}"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok": {
"field": "message",
"patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4}|%{USERDATA5}|%{USERDATA6})\\] %{WORD} - %{COMMAND:mail.command} %{GREEDYDATA:mail.reason}"],
"pattern_definitions" : {
"MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})",
"ACCOUNT": "(%{WORD}|%{EMAIL})",
"USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA2": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA3": "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA4": "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};",
"USERDATA5": "(name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};", "USERDATA6": "(name=%{ACCOUNT:mail.client_name};|)oip=%{IP:mail.remote_ip};oport=%{POSINT:mail.port};oproto=%{WORD:mail.proto};soapId=%{DATA:mail.soap_id};(mid=%{POSINT};ds=%{WORD}};|)",
"COMMAND": "%{WORD}(| %{WORD})",
"LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] index - %{DATA:mail.reason} \\{%{GREEDYDATA:mail.message}\\}"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAILADDRESS})",
"USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}"
},
"ignore_failure" : true
}
},
{
"grok": {
"field": "message",
"patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] nginxlookup - %{DATA:mail.reason}:%{GREEDYDATA:mail.client_name}"],
"pattern_definitions" : {
"MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT": "(%{WORD}|%{EMAIL})",
"USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA2": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};",
"LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok": {
"field": "message",
"patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4}|%{USERDATA5}|%{USERDATA6}|%{USERDATA7})\\] %{WORD} - %{COMMAND:mail.command} %{GREEDYDATA:mail.reason}"],
"pattern_definitions" : {
"MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})",
"ACCOUNT": "(%{WORD}|%{EMAIL})",
"USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(ip=%{IP:mail.ip};|)oip=%{IP:mail.remote_ip};(via=%{DATA:mail.via};|)ua=%{DATA:mail.ua};(cid=%{POSINT:mail.cid};|)",
"USERDATA2": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};(via=%{DATA:mail.via};|)(ua=%{DATA:mail.ua};|)cid=%{POSINT:mail.cid};", "USERDATA3": "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA4": "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};",
"USERDATA5": "(name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.mail.client_account};|)(mid=%{POSINT:mail.mid};|)ip=%{IP:mail.ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};",
"USERDATA7": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};(port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA6": "(name=%{ACCOUNT:mail.client_name};|)oip=%{IP:mail.remote_ip};oport=%{POSINT:mail.port};oproto=%{WORD:mail.proto};soapId=%{DATA:mail.soap_id};(mid=%{POSINT};ds=%{WORD}};|)",
"COMMAND": "%{WORD}(| %{WORD})",
"LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok": {
"field": "message",
"patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2}|%{USERDATA3})\\] mailop - (%{MOVINGMESSAGE}|%{ADDINGMESSAGE}|%{DELETEMESSAGE})"],
"pattern_definitions" : {
"MAILSERVICE": "%{DATA:mail.service}(|-%{POSINT:mail.pid})(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT": "(%{WORD}|%{EMAIL})",
"USERDATA1": "name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};",
"USERDATA2": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(||ip=%{IP:mail.ip};)",
"USERDATA3": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};",
"ADDINGMESSAGE": "%{DATA:mail.command}: id=%{NONNEGINT:mail.id}, Message-ID=<%{DATA:mail.message_id}>, parentId=%{DATA}, folderId=%{NONNEGINT:mail.folder_id}, folderName=%{DATA:mail.folder_name} acct=%{GREEDYDATA:mail.acct}.",
"MOVINGMESSAGE": "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\) to Folder %{DATA:mail.folder_name} \\(id=%{NONNEGINT:mail.folder_id}\\)",
"DELETEMESSAGE": "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\).",
"LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok": {
"field": "message",
"patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA5}|%{USERDATA7})\\] %{COMMAND:mail.command} - %{UPLOAD_COMMAND:mail.commands} %{GREEDYDATA:mail.reason}"],
"pattern_definitions" : {
"MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})",
"ACCOUNT": "(%{WORD}|%{EMAIL})",
"USERDATA5": "(name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)(mid=%{POSINT:mail.mid};|)ip=%{IP:mail.ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};",
"USERDATA7": "name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};(port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(soapId=%{DATA:mail.soap_id};|)",
"COMMAND": "%{WORD}",
"LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"UPLOAD_COMMAND": "(Received plain: Upload:|saveUpload\\(\\): received Upload:)",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - %{REASON:mail.reason}"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};",
"REASON" : "Account is lockout, %{GREEDYDATA}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - cmd=%{DATA:mail.command}; account=%{ACCOUNT:mail.client_account}; protocol=%{WORD:mail.protocol}; error=%{DATA:mail.error} \\[%{EMAIL:mail.client_name}\\], %{REASON:mail.reason};"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"REASON" : "%{DATA}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
}
]
}

41
pipelines/maillog-with-geoip.json 100644
View File

@ -0,0 +1,41 @@
{
"processors": [
{
"pipeline" : {
"name": "maillog"
}
},
{
"pipeline" : {
"name": "amavis"
}
},
{
"pipeline" : {
"name": "mailboxlog"
}
},
{
"pipeline" : {
"name": "zimbralog"
}
},
{
"pipeline" : {
"name": "fail2ban"
}
},
{
"geoip" : {
"field" : "mail.remote_ip",
"ignore_missing" : true
}
},
{
"geoip" : {
"field" : "f2b.remote_ip",
"ignore_missing" : true
}
}
]
}

774
pipelines/maillog.json 100644
View File

@ -0,0 +1,774 @@
{
"processors" : [
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{MAILSERVICE:mail.service}-%{NONNEGINT:mail.pid}\\] \\[name=%{EMAIL:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.mail.remote_ip};via=%{IP:mail.relay_ip}\\(%{DATA}\\);%{DATA}\\] %{DATA} - %{REASON:mail.reason}"
],
"pattern_definitions" : {
"RELAYPORT" : "[0-9]+",
"REASON" : "(?:.+)",
"SEVERITY" : "(?:.+)",
"CONNECTIONSTATUS" : "connect|disconnect",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "(?:.+)",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ID" : "(?:[0-9]+)",
"CLIENTDATA" : "name=%{EMAIL:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"REMOTEPORT" : "[0-9]+",
"SOAPID" : "soapId=%{DATA:mail.soap_id}",
"PROTOCOL" : "%{WORD}",
"SEVERITY" : "(?:.+)",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"ID" : "(?:[0-9]+)",
"SOAPID" : "soapId=%{WORD:mail.soap_id}",
"CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};ua=%{DATA:mail.ua};%{SOAPID};",
"REMOTEPORT" : "[0-9]+",
"PROTOCOL" : "%{WORD}",
"SEVERITY" : "(?:.+)",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"ID" : "(?:[0-9]+)",
"CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};",
"REMOTEPORT" : "[0-9]+",
"SOAPID" : "soapId=%{DATA:mail.soap_id}",
"SEVERITY" : "(?:.+)",
"PROTOCOL" : "%{WORD}",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"ID" : "(?:[0-9]+)",
"CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};ua=%{DATA:mail.ua};%{SOAPID};",
"REMOTEPORT" : "[0-9]+",
"SOAPID" : "soapId=%{WORD:mail.soap_id}",
"PROTOCOL" : "%{WORD}",
"SEVERITY" : "(?:.+)",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol}; %{ERRORMESSAGE};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"ID" : "(?:[0-9]+)",
"CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};",
"REMOTEPORT" : "[0-9]+",
"SOAPID" : "soapId=%{DATA:mail.soap_id}",
"PROTOCOL" : "%{WORD}",
"SEVERITY" : "(?:.+)",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"ERRORMESSAGE" : "error=%{DATA:mail.error}( \\[%{ACCOUNT}\\]|), %{DATA:mail.reason}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};"
],
"pattern_definitions" : {
"URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})",
"ID" : "(?:[0-9]+)",
"CLIENTDATA" : "name=%{EMAIL:mail.client_name};oip=%{IP:mail.remote_ip};ua=%{DATA:mail.ua};%{SOAPID};",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"REMOTEPORT" : "[0-9]+",
"SOAPID" : "soapId=%{DATA:mail.soap_id}",
"PROTOCOL" : "%{WORD}",
"SEVERITY" : "(?:.+)",
"COMMAND" : "%{DATA} cmd=%{WORD:mail.command}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - cmd=%{COMMAND:mail.command}; account=%{ACCOUNT:mail.client_name}; protocol=%{WORD:mail.protocol}"
],
"pattern_definitions" : {
"MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})",
"ACCOUNT" : "(%{WORD}|%{EMAIL})",
"USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};",
"USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};",
"USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};",
"COMMAND" : "%{WORD}",
"LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: to=<%{DATA:mail.to}>,(?:\\sorig_to=<%{EMAIL:mail.orig_to}>,)? relay=%{RELAY}, delay=%{NUMBER:mail.delay}, delays=%{DELAYS}, dsn=%{DSN:mail.dsn}, status=%{STATUS:mail.status} \\(%{DATA:mail.reason}\\)"
],
"pattern_definitions" : {
"RELAY" : "(?:%{HOSTNAME:mail.relay_host}(?:\\[%{IP:mail.relay_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}",
"PERMERROR" : "5[0-9]{2}",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"STATUS" : "sent|deferred|bounced|expired",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{POSTFIXACTION:mail.postfix_action}: %{DATA:mail.command} from %{REMOTE}: %{PERMERROR:mail.response_code} %{DSN:mail.dsn} %{DATA}: %{DATA:mail.reason}; (from=<%{EMAIL:mail.from}> |)to=<%{EMAIL:mail.to}> proto=%{DATA:mail.proto} helo=<%{HELO}>"
],
"pattern_definitions" : {
"POSTFIXACTION" : "discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"PERMERROR" : "(4|5)[0-9][0-9]",
"DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"HELO" : "(?:\\[%{IP:mail.helo}\\]|%{HOSTNAME:mail.helo}|%{DATA:mail.helo})",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{MESSAGELEVEL:mail.message_level}: hostname %{HOSTNAME:mail.remote_host} %{DATA:\n } address %{IP:mail.remote_ip}"
],
"pattern_definitions" : {
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"MESSAGELEVEL" : "reject|warning|error|fatal|panic",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} %{DIRECTION:mail.connect_direction} %{REMOTE}:%{REMOTEPORT:mail.remote_port}: %{REASON:mail.reason}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"REMOTEPORT" : "[0-9]+",
"REASON" : "(?:.+)",
"CONNECTIONSTATUS" : "connect|disconnect",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"DIRECTION" : "(to|from)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} %{DIRECTION:mail.connect_direction} %{REMOTE}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"REMOTEPORT" : "[0-9]+",
"REASON" : "(?:.+)",
"CONNECTIONSTATUS" : "connect|disconnect",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"DIRECTION" : "(to|from)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} from \\[%{IPORHOST:mail.remote_host}\\]:%{PORT:mail.remote_port} to \\[%{IPORHOST:mail.local_host}\\]:%{PORT:mail.local_port}"
],
"pattern_definitions" : {
"PORT" : "[0-9]+",
"CONNECTIONSTATUS" : "CONNECT|DISCONNECT|connect|disconnect",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} after %{DATA:mail.command} from %{REMOTE}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"CONNECTIONSTATUS" : "lost connection",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{REMOTE}, sasl_method=%{DATA:mail.sasl_method}, sasl_username=%{EMAIL:mail.username}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: message-id=(|<)%{DATA:mail.message_id}(|>)"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: from=<%{EMAIL:mail.from}>, size=%{DATA:mail.size}, nrcpt=%{DATA:mail.nrcpt} \\(%{DATA:mail.reason}\\)"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{MESSAGESTATUS:mail.reason}"
],
"pattern_definitions" : {
"MESSAGESTATUS" : "removed",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: (%{HOSTDATA}:\\s|\\s)%{DATA:mail.error}: %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"HOSTDATA" : "%{HOSTNAME:mail.remote_host}\\[%{IPORHOST:mail.remote_ip}\\]",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"SEVERITY" : "(warning|info|error)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{DATA:mail.reason} \\[%{IP:mail.remote_ip}\\]:%{NONNEGINT:mail.remote_port}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{DATA:mail.reason} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]: TLS%{DATA:mail.tls_proto} \\(%{DATA}\\)"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}(|:) %{CONNECTIONSTATUS:mail.connection_status} from %{REMOTE} ehlo=%{NONNEGINT:mail.ehlo} mail=%{DATA} rcpt=%{NONNEGINT:mail.rcpt} data=%{WORD:mail.data} (noop=%{NONNEGINT:mail.noop} |)quit=%{NONNEGINT:mail.quit} commands=%{NONNEGINT:mail.commands}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"CONNECTIONSTATUS" : "connect|disconnect",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: hostname %{HOSTNAME:mail.remote_host} %{DATA:mail.reason} address %{IP:mail.remote_ip}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"SEVERITY" : "(warning|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{SASL:mail.sasl_message}: %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"SASL" : "SASL %{DATA}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"SEVERITY" : "(warning|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{POSTFIXACTION:mail.postfix_action}: %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"POSTFIXACTION" : "statistics",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{POSTFIXACTION:mail.postfix_action} %{GREEDYDATA:mail.reason}: retained=%{NONNEGINT:mail_cache_retained} dropped=%{NONNEGINT:mail.cache_dropped} entries"
],
"pattern_definitions" : {
"POSTFIXACTION" : "cache",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{MSG:mail.reason}: %{QUEUEID:mail.qid2}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"MSG" : "(sender (non-delivery|delivery status) notification)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: resent-message-id=<%{DATA:mail.message_id}>"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"MSG" : "sender non-delivery notification"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: host %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] said: %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: conversation with %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: from=<%{EMAIL:mail.from}>, status=%{WORD:mail.status}, %{GREEDYDATA:mail.reason}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: host %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] %{STATUS:mail.status} %{DATA}: %{DATA:mail.reason};"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"STATUS" : "refused",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}(:|) %{QUEUEID:mail.qid}: %{POSTFIXACTION:mail.postfix_action}: %{DATA:mail.command} from %{REMOTE}: <%{EMAIL:mail.client}>: %{DATA:mail.reason}; from=<%{EMAIL:mail.from}> to=<%{EMAIL:mail.to}> proto=%{DATA:mail.proto} helo=<%{HELO}>"
],
"pattern_definitions" : {
"POSTFIXACTION" : "discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"HELO" : "(?:\\[%{IP:mail.helo}\\]|%{HOSTNAME:mail.helo}|%{DATA:mail.helo})",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE} %{QUEUEID:mail.qid}: client=%{REMOTE}"
],
"pattern_definitions" : {
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{STATUS:mail.connection_status}: %{DATA:mail.reason} from %{REMOTE}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"STATUS" : "reject",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"SEVERITY" : "(warning|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{REASON:mail.reason} after %{DATA:mail.command} from %{REMOTE}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"REASON" : "((too many errors)|(timeout))",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} from %{REMOTE} in %{DATA:mail.command} command: %{GREEDYDATA:mail.message}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"STATUS" : "reject",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)",
"SEVERITY" : "(warning|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{DATA:mail.reason} for %{REMOTE}$"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"STATUS" : "reject",
"REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)",
"HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} for %{HOSTNAME:mail.remote_host}: %{IP:mail.remote_ip}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"SEVERITY" : "(warning|error|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} (from|with) %{HOSTDATA}%{GREEDYDATA:mail.message}"
],
"pattern_definitions" : {
"QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)",
"HOSTDATA" : "%{HOSTNAME:mail.remote_host}\\[%{IPORHOST:mail.remote_ip}\\]",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}",
"MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"SEVERITY" : "(warning|info)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{IP:mail.remote_ip}:%{POSINT:mail.remote_port} -%{REMOTEUSER}- \\[%{LOGDATETIME:mail.log_datetime} %{ISO8601_TIMEZONE}\\](\\s.+?|)\\\"%{REQUEST}\\ %{POSINT:mail.http_status} %{NONNEGINT:mail.http_bytes_sent} \\\"%{REFERER:mail.http_referer}\\\" \\\"%{DATA:mail.http_user_agent}\\\" \\\"%{IP:mail.ip1}:%{POSINT:mail.port1}\\\" (\\\"%{IP:mail.ip2}:%{POSINT:mail.port2}\\\")"
],
"pattern_definitions" : {
"REASON" : "(?:.+)",
"REMOTEUSER" : "(\\s|%{DATA:mail.remote_user})",
"REQUEST" : "%{DATA:mail.http_method} %{URI:mail.http_request_url} %{DATA}",
"REFERER" : "%{URI}|-",
"SEVERITY" : "(?:.+)",
"LOGDATETIME" : "%{MONTHDAY}/%{MONTH}/20%{YEAR}:%{TIME}"
},
"ignore_failure" : true
}
},
{
"date_index_name" : {
"field" : "@timestamp",
"index_name_prefix" : "maillog_filebeat-",
"date_rounding" : "d",
"index_name_format" : "yyyy-MM-dd"
}
}
]
}

89
pipelines/zimbralog.json 100644
View File

@ -0,0 +1,89 @@
{
"processors": [
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTION:mail.connection_status} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\](%{DATA:mail.reason}|)$"
],
"pattern_definitions" : {
"HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"CONNECTION" : "(connect|disconnect)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]"
],
"pattern_definitions" : {
"HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"QUEUED" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: filter: %{DATA} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]: %{DATA}; from=<%{EMAIL:mail.from}> to=<%{EMAIL:mail.to}> proto=%{DATA:mail.protocol} helo=<%{HOSTNAME:mail.helo}>"
],
"pattern_definitions" : {
"HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)",
"PORT" : "(?:[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"QUEUED" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: to=<%{EMAIL:mail.to}>, (|(orig_to=<%{EMAIL:mail.orig_to}>, ))((relay=%{HOSTNAME:mail.relay_host}\\[%{IP:mail.relay_ip}\\]:%{PORT:mail.relay_port})|(relay=%{WORD:mail.relay_host})), (conn_use=%{WORD}, |)delay=%{NUMBER:mail.delay}, delays=%{DELAYS}, dsn=%{DSN:mail.dsn}, status=%{WORD:mail.status} \\(%{DATA:mail.reason}\\)((: %{DATA}: queued as %{QUEUED:mail.qid2})|)$"
],
"pattern_definitions" : {
"HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)",
"PORT" : "(?:[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}",
"DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}",
"QUEUED" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "message",
"patterns" : [
"%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: from=<%{EMAIL:mail.from}>, size=%{WORD:mail.size}, nrcpt=%{WORD} \\(%{DATA:mail.reason}\\)$"
],
"pattern_definitions" : {
"HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)",
"PORT" : "(?:[0-9]+)",
"LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}",
"MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]",
"EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)",
"DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}",
"DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}",
"QUEUED" : "(?:[A-F0-9]+|NOQUEUE)"
},
"ignore_failure" : true
}
}
]
}

3
scripts/README.md 100644
View File

@ -0,0 +1,3 @@
# Вспомогательные скрипты и утилиты для ElasticSearch
- elastic_idex_del.pl - удаление индексов ES старше определоенной даты

80
scripts/elastic_index_del.pl 100755
View File

@ -0,0 +1,80 @@
#!/usr/bin/perl
#########################################
#
# ElasticSearch index remover
#
# Author: Sergey Kalinin
#
# https://nuk-svk.ru
# svk@nuk-svk.ru
#########################################
use Getopt::Long;
use vars qw(%opts);
use feature qw(say);
my $help = 0;
my $debug = 0;
my $domain_file;
GetOptions(\%opts, 'host=s', 'days=s', 'data-type=s', 'help', 'verbose');
if (defined($opts{'help'}))
{
print STDERR <<EOT;
ElastickSearch index remover
Usage: $0 --host localhost --days (days) --data-type type-of-data [--verbose --help]
--host - elasticsearch(kibana) host name or ip address
--days - days ago count
--data-type - must be like kibana index (netflow, log4j, logback, etc)
--verbose - output debug information
--help - print this message
EOT
exit 1;
}
if ( $opts{'host'} eq "" ) {
print "Wrong host: $opts{'host'}\n";
exit;
} else {
$host = $opts{'host'};
}
if ( $opts{'days'} eq "" || $opts{'days'} !~ /^\d+$/ ) {
print "Wrong days count: $opts{'days'}\n";
exit;
} else {
$days = $opts{'days'};
}
if ( $opts{'data-type'} eq "" ) {
print "Wrong data type: $opts{'data-types'}\n";
exit;
} else {
$data_type = $opts{'data-type'};
}
$date = `date --date="-$days day" +%Y.%m.%d`;
$date =~ s/\s//g;
# get neflow index from kibana
#say "curl -s -XGET \"http://$host:9200/_cat/indices?v&pretty\" -H 'Content-Type: application/json' | grep $data_type";
my $data_list = `curl -s -XGET "http://$host:9200/_cat/indices?v&pretty" -H 'Content-Type: application/json' | grep $data_type`;
foreach my $index (split('\n',$data_list)) {
#$index =~ /netflow-(\d{4}\.\d{2}\.\d{2})/;
$index =~ /$data_type-(\d{4}\.\d{2}\.\d{2})/;
$index_date = $1;
if ($index_date eq $date) {
$del_cmd = "curl -s -XDELETE 'http://$host:9200/$data_type-$index_date' -H 'Content-Type: application/json'";
say $del_cmd;
$res = `$del_cmd`;
say "Result: $res" if $res;
}
}

261
zabbix_template/Template_App_Elasticsearch.xml 100755
View File

@ -0,0 +1,261 @@
<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
<version>5.0</version>
<date>2020-12-30T12:08:33Z</date>
<groups>
<group>
<name>Elastic cluster</name>
</group>
</groups>
<templates>
<template>
<template>Template Elasticsearch</template>
<name>Template Elasticsearch</name>
<groups>
<group>
<name>Elastic cluster</name>
</group>
</groups>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
<application>
<name>Elasticsearch https access</name>
</application>
</applications>
<items>
<item>
<name>Кластер Elasticsearch</name>
<key>es.cluster</key>
<trends>0</trends>
<value_type>TEXT</value_type>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
</item>
<item>
<name>Количество основных шард</name>
<type>DEPENDENT</type>
<key>es.cluster.active_primary_shards</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.active_primary_shards</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество активных шард</name>
<type>DEPENDENT</type>
<key>es.cluster.active_shards</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.active_shards</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество активных шард в процентах</name>
<type>DEPENDENT</type>
<key>es.cluster.active_shards_percent_as_number</key>
<delay>0</delay>
<value_type>FLOAT</value_type>
<units>%</units>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.active_shards_percent_as_number</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество отложенных неназначенных шард</name>
<type>DEPENDENT</type>
<key>es.cluster.delayed_unassigned_shards</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.delayed_unassigned_shards</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество data node</name>
<type>DEPENDENT</type>
<key>es.cluster.number_of_data_nodes</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.number_of_data_nodes</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество узлов</name>
<type>DEPENDENT</type>
<key>es.cluster.number_of_nodes</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.number_of_nodes</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество отложенных задач</name>
<type>DEPENDENT</type>
<key>es.cluster.number_of_pending_tasks</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.number_of_pending_tasks</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество перемещаемых шард</name>
<type>DEPENDENT</type>
<key>es.cluster.relocating_shards</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.relocating_shards</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Статус кластера</name>
<type>DEPENDENT</type>
<key>es.cluster.status</key>
<delay>0</delay>
<trends>0</trends>
<value_type>TEXT</value_type>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.status</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
<item>
<name>Количество неназначенных шард</name>
<type>DEPENDENT</type>
<key>es.cluster.unassigned_shards</key>
<delay>0</delay>
<applications>
<application>
<name>Elasticsearch cluster status</name>
</application>
</applications>
<preprocessing>
<step>
<type>JSONPATH</type>
<params>$.unassigned_shards</params>
</step>
</preprocessing>
<master_item>
<key>es.cluster</key>
</master_item>
</item>
</items>
<httptests>
<httptest>
<name>Elasticsearch https access</name>
<application>
<name>Elasticsearch https access</name>
</application>
<steps>
<step>
<name>https://{HOST.HOST}:9200</name>
<url>https://{HOST.HOST}:9200</url>
<follow_redirects>NO</follow_redirects>
</step>
</steps>
</httptest>
</httptests>
</template>
</templates>
</zabbix_export>