commit c3c29925cfff47d527b57a51bb34d1f038e7fd67 Author: Sergey Kalinin Date: Fri Jan 15 18:19:45 2021 +0300 Initial release diff --git a/README.md b/README.md new file mode 100644 index 0000000..07fc7f3 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Что тут + +Тут лежат всё что связано с elasticsearch + +# Состав + +- cerebro - это ВЭБ-сервис для управления ES. В каталоге скрипты для запуска cerebro (https://github.com/lmenezes/cerebro) в docker +- curator - всё для сборки и запуска сервиса в docker. Curator это сервис управления индексами elasticsearch https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html +- pipelines - шаблоны для разборки различных данных в elasticsearch ingestnode. +- scripts - разные вспомогательные скрипты +- zabbix_template - шаблон для мониторинга ES-кластера \ No newline at end of file diff --git a/cerebro/.env b/cerebro/.env new file mode 100644 index 0000000..3a9fe17 --- /dev/null +++ b/cerebro/.env @@ -0,0 +1,4 @@ +NGINX_HOST=elk.example.com + +CEREBRO_PASSWORD=some_password +CEREBRO_USER=admin \ No newline at end of file diff --git a/cerebro/Dockerfile b/cerebro/Dockerfile new file mode 100644 index 0000000..39aa5bb --- /dev/null +++ b/cerebro/Dockerfile @@ -0,0 +1,21 @@ +FROM openjdk:11-jre-slim + +ENV CEREBRO_VERSION 0.9.2 + +RUN apt-get update \ + && apt-get install -y wget \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /opt/cerebro/logs \ + && wget -qO- https://github.com/lmenezes/cerebro/releases/download/v${CEREBRO_VERSION}/cerebro-${CEREBRO_VERSION}.tgz \ + | tar xzv --strip-components 1 -C /opt/cerebro \ + && sed -i '//d' /opt/cerebro/conf/logback.xml \ + && addgroup -gid 1000 cerebro \ + && adduser -gid 1000 -uid 1000 cerebro \ + && chown -R cerebro:cerebro /opt/cerebro + +RUN echo "play.ws.ssl.loose.acceptAnyCertificate = true" >> /opt/cerebro/conf/reference.conf + +WORKDIR /opt/cerebro +USER cerebro + +ENTRYPOINT [ "/opt/cerebro/bin/cerebro" ] \ No newline at end of file diff --git a/cerebro/docker-compose.yml b/cerebro/docker-compose.yml new file mode 100644 index 0000000..3776760 --- /dev/null +++ b/cerebro/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' +services: + cerebro: + image: cerebro:latest + container_name: cerebro + build: + context: ./ + restart: always + # ports: + # - 5555:5000 + # expose: + # - "5555" + environment: + - AUTH_TYPE=basic + - BASIC_AUTH_USER=${CEREBRO_USER:-admin} + - BASIC_AUTH_PWD=${CEREBRO_PASSWORD:-admin} + networks: + - odfe-net + depends_on: + - nginx + + nginx: + image: nginx + container_name: nginx + env_file: + - .env + restart: always + ports: + - 443:443 + environment: + - NGINX_HOST=${NGINX_HOST} + volumes: + - ./nginx/templates:/etc/nginx/templates + - ./ssl/node.pem:/etc/nginx/certs/nginx-selfsigned.pem:ro + - ./ssl/node.key:/etc/nginx/certs/nginx-selfsigned.key:ro + networks: + - odfe-net \ No newline at end of file diff --git a/cerebro/nginx/templates/elastic.conf.template b/cerebro/nginx/templates/elastic.conf.template new file mode 100644 index 0000000..3783e9d --- /dev/null +++ b/cerebro/nginx/templates/elastic.conf.template @@ -0,0 +1,15 @@ +server { + listen 443 ssl; + server_name ${NGINX_HOST}; + client_max_body_size 100M; + ssl_certificate /etc/nginx/certs/nginx-selfsigned.pem; + ssl_certificate_key /etc/nginx/certs/nginx-selfsigned.key; + + location / { + proxy_pass http://cerebro:9000; + # proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file diff --git a/curator/.env b/curator/.env new file mode 100644 index 0000000..9e79fc6 --- /dev/null +++ b/curator/.env @@ -0,0 +1,2 @@ +ELASTIC_HOST=elk.example.com +CURATOR=admin:some_password diff --git a/curator/Dockerfile b/curator/Dockerfile new file mode 100644 index 0000000..6d7f44e --- /dev/null +++ b/curator/Dockerfile @@ -0,0 +1,18 @@ +FROM debian:buster-slim + +RUN apt update && apt install -y wget gnupg2 +RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - +RUN echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian9 stable main" > /etc/apt/sources.list.d/curator.list +RUN apt update -y && apt install -y cron elasticsearch-curator tini + +ENV CURATOR_VERSION=5.8.1 +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +COPY entrypoint.sh / + +RUN ["chmod", "+x", "/entrypoint.sh"] + +WORKDIR /usr/share/curator + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/curator/config/action_delete_alerts.yml b/curator/config/action_delete_alerts.yml new file mode 100644 index 0000000..95c3989 --- /dev/null +++ b/curator/config/action_delete_alerts.yml @@ -0,0 +1,18 @@ +actions: + 5: + action: delete_indices + description: >- + Delete indices. Find which to delete by first limiting the list to logstash- + prefixed indices. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: .opendistro-alerting- + - filtertype: age + source: creation_date + direction: older + unit: days + unit_count: 10 diff --git a/curator/config/action_delete_filebeat.yml b/curator/config/action_delete_filebeat.yml new file mode 100644 index 0000000..f49e296 --- /dev/null +++ b/curator/config/action_delete_filebeat.yml @@ -0,0 +1,17 @@ +actions: + 4: + action: delete_indices + description: >- + Delete indices older than + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: filebeat- + - filtertype: age + source: creation_date + direction: older + unit: days + unit_count: 10 diff --git a/curator/config/action_delete_logstash.yml b/curator/config/action_delete_logstash.yml new file mode 100644 index 0000000..61d4644 --- /dev/null +++ b/curator/config/action_delete_logstash.yml @@ -0,0 +1,18 @@ +actions: + 1: + action: delete_indices + description: >- + Delete indices. Find which to delete by first limiting the list to logstash- + prefixed indices. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: creation_date + direction: older + unit: days + unit_count: 2 diff --git a/curator/config/action_delete_maillog.yml b/curator/config/action_delete_maillog.yml new file mode 100644 index 0000000..820faa8 --- /dev/null +++ b/curator/config/action_delete_maillog.yml @@ -0,0 +1,17 @@ +actions: + 7: + action: delete_indices + description: >- + Delete indices older than + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: maillog_filebeat- + - filtertype: age + source: creation_date + direction: older + unit: days + unit_count: 30 diff --git a/curator/config/action_delete_winlogbeat.yml b/curator/config/action_delete_winlogbeat.yml new file mode 100644 index 0000000..436973b --- /dev/null +++ b/curator/config/action_delete_winlogbeat.yml @@ -0,0 +1,18 @@ +actions: + 3: + action: delete_indices + description: >- + Delete indices. Find which to delete by first limiting the list to logstash- + prefixed indices. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: winlogbeat- + - filtertype: age + source: creation_date + direction: older + unit: days + unit_count: 10 diff --git a/curator/config/curator.yml b/curator/config/curator.yml new file mode 100644 index 0000000..35a36bd --- /dev/null +++ b/curator/config/curator.yml @@ -0,0 +1,24 @@ +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +client: + hosts: + - ${ELASTICSEARCH_HOST} + port: 9200 + url_prefix: + use_ssl: True + certificate: /etc/cert.pem + client_cert: + client_key: + aws_key: + aws_secret_key: + aws_region: + ssl_no_validate: True + http_auth: ${CURATOR} + timeout: 30 + master_only: False + +logging: + loglevel: INFO + logfile: + logformat: default diff --git a/curator/docker-compose.yml b/curator/docker-compose.yml new file mode 100644 index 0000000..12ade85 --- /dev/null +++ b/curator/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' +services: + curator: + build: + context: curator/ + image: curator:latest + restart: always + container_name: curator + env_file: + - .env + environment: + ELASTICSEARCH_HOST: ${ELASTIC_HOST} + CRON: "0 0 * * *" + volumes: + - ./curator/config:/usr/share/curator/config + - ./ssl/MyRootCA.pem:/etc/cert.pem:ro + networks: + - odfe-net diff --git a/curator/entrypoint.sh b/curator/entrypoint.sh new file mode 100755 index 0000000..1b5ff52 --- /dev/null +++ b/curator/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +#shopt -s nullglob +FILES=/usr/share/curator/config/action_* + +for f in $FILES +do + echo "$CRON /usr/bin/curator --config /usr/share/curator/config/curator.yml $f" >> /etc/crontab +done + +# https://github.com/krallin/tini/blob/master/README.md#subreaping +tini -s -- cron -f -L 8 \ No newline at end of file diff --git a/pipelines/README.md b/pipelines/README.md new file mode 100644 index 0000000..a0bf5f1 --- /dev/null +++ b/pipelines/README.md @@ -0,0 +1,14 @@ +# Шаблоны для создания pipelines в ingestnode ES. + +- amavis.json - обработка логов amavis +- cdr.json - разбор логов от АТС (avaya, asterisk) +- fail2ban.json - разбор лога F2B +- mailboxlog.json - обработка лога mailboxlog от Zimbra +- maillog.json - шаблон для обработки логов (maillog, fail2ban.log, zibmra/audit.log, zimbra/nginx.access.log) +- maillog_with_geoip.json - шаблон pipeline для получения GEO данных, на основе полей создаваемых предыдущим (maillog) (используется последовательный запуск) +- zaimbralog.json - парсинг почтового лога Zimbra + +# Внесение изменений в elasticsearch + +```curl -X POST "elk.example.com:9200"/_ingest/pipeline/maillog" -H 'Content-Type: application/json' -d '@maillog.json'``` +```curl -X POST "elk.example.com:9200"/_ingest/pipeline/maillog_with_geoip" -H 'Content-Type: application/json' -d '@maillog_with_geoip.json'``` diff --git a/pipelines/amavis.json b/pipelines/amavis.json new file mode 100644 index 0000000..efb9b86 --- /dev/null +++ b/pipelines/amavis.json @@ -0,0 +1,188 @@ +{ + "processors": [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{WORD:mail.command}: %{WORD:mail.amavis_result} %{DATA:mail.reason}, From: <%{EMAIL:mail.from}>" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result}, <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ %{DATA:mail.amavis_spam_tag}, score=%{DATA:mail.amavis_spam_score} required=%{DATA} tests=\\[%{GREEDYDATA:mail.amavis_spam_result}\\]" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} %{DATA:mail.amavis_result} from <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ %{DATA:mail.reason}: queued as %{QUEUEID:mail.qid}" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, %{DATA:mail.reason} \\[%{IP:mail.host}\\]:%{PORT:mail.port} <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, queued_as: %{QUEUEID:mail.amavis_qid}, dkim_sd=%{DATA:mail.dkim}, %{POSINT:mail.amavis_delay} ms" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} :%{PORT:mail.port} %{DATA:mail.amavis_file}: <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}> (SIZE=%{POSINT:mail.size} |)(BODY=%{WORD} |)%{DATA:mail.amavis_result}: from %{HOSTNAME:mail.remote_host} \\(\\[%{IP:mail.remote_ip}\\]\\) by %{DATA} \\(%{HOSTNAME:mail.host} \\[%{IP:mail.ip}\\]\\) \\(%{DATA}\\) %{DATA:mail.reason}; %{GREEDYDATA:mail.amavis_datetime}" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result}: %{DATA:mail.amavis_qid} (%{DATA:mail.reason} |)\\[%{IP:mail.remote_ip}\\] <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{WORD:mail.amavis_qid} %{DATA:mail.amavis_result} from <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, %{DATA} from %{WORD}\\(%{WORD}:\\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port}\\): %{DATA:mail.reason}: queued as %{WORD:mail.qid}" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, \\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port} \\[%{IP}\\] <%{EMAIL:mail.from}> -> (<%{EMAIL:mail.to}>,)+ Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, (queued_as: %{QUEUEID:mail.amavis_qid}, |)%{POSINT:mail.amavis_delay} ms" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA:mail.amavis_result} \\{%{DATA:mail.amavis_way}\\}, %{DATA:mail.reason} \\[%{IP:mail.remote_ip}\\]:%{PORT:mail.remote_port} <%{EMAIL:mail.from}> -> <%{EMAIL:mail.to}>, Queue-ID: %{QUEUEID:mail.qid}, Message-ID: <%{DATA:mail.message_id}>, mail_id: %{DATA:mail.id}, Hits: %{DATA:mail.amavis_hits}, size: %{POSINT:mail.size}, queued_as: %{QUEUEID:mail.amavis_qid}, %{POSINT:mail.amavis_delay} ms" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) size: %{POSINT:mail.size}, %{WORD:mail.amavis_result} \\[total %{POSINT:mail.amavis_check.total} ms, cpu %{POSINT:mail.amavis_check.cpu} ms((, AM-cpu %{POSINT} ms)|)((, SA-cpu %{POSINT} ms)|)\\] - SMTP greeting: %{BASE10NUM:mail.amavis_check.smtp_greeting} \\(%{PORT:mail.amavis_check.smtp_greeting_percent}\\%\\)%{WORD}, SMTP EHLO: %{BASE10NUM:mail.amavis_check.smtp_ehlo} \\(%{PORT:mail.amavis_check.smtp_ehlo_percent}\\%\\)%{WORD}, SMTP pre-MAIL: %{BASE10NUM:mail.amavis_check.smtp_pre_mail} \\(%{PORT:mail.amavis_check.smtp_pre_mail_percent}\\%\\)%{WORD}((, lookup_ldap: %{BASE10NUM:mail.amavis_check.lookup_ldap} \\(%{PORT:mail.amavis_check.lookup_ldap_percent}\\%\\)%{WORD})+), SMTP pre-DATA-flush: %{BASE10NUM:mail.amavis_check.smtp_pre_data_flush} \\(%{PORT:mail.amavis_check.smtp_pre_data_flush_percent}\\%\\)%{WORD}, SMTP DATA: %{BASE10NUM:mail.amavis_check.smtp_data} \\(%{PORT:mail.amavis_check.smtp_data_percent}\\%\\)%{WORD}, %{DATA}mime_decode: %{BASE10NUM:mail.amavis_check.mime_decode} \\(%{PORT:mail.amavis_check.mime_decode_percent}\\%\\)%{WORD}, %{DATA}SMTP pre-response: %{BASE10NUM:mail.amavis_check.smtp_pre_response} \\(%{PORT:mail.amavis_check.smtp_pre_response_percent}\\%\\)%{WORD}, SMTP response: %{BASE10NUM:mail.amavis_check.smtp_response} \\(%{PORT:mail.amavis_check.smtp_response_percent}\\%\\)%{WORD}" + ], + "pattern_definitions" : { + "PORT" : "(?:[0-9]+)", + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: \\(%{PID2:mail.pid2}\\) %{DATA} Content-Type: %{DATA:mail.content_type}(((, size: %{DATA:mail.size} B, name:)|)(( %{GREEDYDATA:mail.file_name})|)|)$" + ], + "pattern_definitions" : { + "PID2" : "(?:[0-9]+-[0-9]+(-[0-9]+|))", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{WORD:mail.service}\\[%{POSINT:mail.pid}\\]" + }, + "ignore_failure" : true + } + + } + ] +} \ No newline at end of file diff --git a/pipelines/cdr.json b/pipelines/cdr.json new file mode 100644 index 0000000..8e38a24 --- /dev/null +++ b/pipelines/cdr.json @@ -0,0 +1,118 @@ +{ + "processors" : [ +{ + "set": { + "field": "cdr.hour", + "value": "0" + } +}, +{ + "set": { + "field": "cdr.minute", + "value": "0" + } +}, +{ + "set": { + "field": "cdr.second", + "value": "0" + } +}, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:cdr.call_start},%{TIME_CUSTOM:cdr.call_duration_time},%{INT:cdr.ring_duration},%{DATA:cdr.caller_number},%{WORD:cdr.direction},%{DATA:cdr.called_number},%{DATA:cdr.dialed_number},%{DATA:cdr.account},%{INT:cdr.is_internal},%{INT:cdr.call_id},%{DATA:cdr.continuation},%{DATA:cdr.party1device},%{DATA:cdr.party1name},%{DATA:cdr.party2device},%{DATA:cdr.party2name},%{DATA:cdr.holdtime},%{DATA:cdr.park_time},%{DATA:cdr.field_1},%{DATA:cdr.field_2},%{DATA:cdr.field_3},%{DATA:cdr.field_4},%{DATA:cdr.field_5},%{DATA:cdr.field_6},%{DATA:cdr.field_7},%{DATA:cdr.field_8},%{DATA:cdr.field_9},%{DATA:cdr.field_10},%{DATA:cdr.field_11},%{DATA:cdr.field_12},%{DATA:cdr.field_13},%{HOST:cdr.hostname},%{DATA:cdr.field_14},%{HOST:cdr.field_15},%{DATA:cdr.field_16},%{LOGDATE}" + ], + "pattern_definitions" : { + "TIME_CUSTOM" : "%{HOUR_CUSTOM:cdr.hour}:%{MINUTE:cdr.minute}(?::%{SECOND_CUSTOM:cdr.second})", + "HOUR_CUSTOM" : "(?!<[0-9])%{HOUR}", + "SECOND_CUSTOM" : "(?:(?:[0-5]?[0-9]|60))", + "HOST": "%{HOSTNAME}|%{IP}", + "LOGDATE" : "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "(<%{DATA}>|)%{LOGDATE:cdr.log_date} %{HOST:cdr.hostname} cdr-master: \"%{ACCOUNT:cdr.account}\",\"%{POSINT:cdr.caller_number}\",\"%{WORD:cdr.called_number}\",%{DATA:cdr.direction},\"%{CALL_DATE:cdr.call_start}\",\"(%{DATA}|)\",\"%{CALL_DATE:cdr.call_end}\",\"%{INT:cdr.call_duration}\",\"%{DATA}\",\"%{DATA:cdr.call_status}\"" + ], + "pattern_definitions" : { + "TIME_CUSTOM": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND_CUSTOM})", + "SECOND_CUSTOM": "(?:(?:[0-5]?[0-9]|60))", + "LOGDATE": "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "HOST": "%{HOSTNAME}|%{IP}", + "CALL_DATE": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}(\\s+)%{TIME}", + "ACCOUNT": "\"(%{DATA}|)\" <%{WORD}>" + }, + "ignore_failure" : true + + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "(<%{DATA}>|)%{LOGDATE:cdr.log_date} %{HOST:cdr.hostname} cdr-master: \"%{INT:cdr.call_duration}\",\"%{WORD:cdr.billsec}\",\"%{DATA:cdr.caller_number}\",\"%{DATA:cdr.channel}\",\"%{WORD:cdr.called_number}\",\"%{WORD:cdr.dialed_number}\",\"%{DATA}\",\"%{DATA:cdr.dst_channel}\",\"%{DATA:cdr.call_id}\"" + ], + "pattern_definitions" : { + "TIME_CUSTOM": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND_CUSTOM})", + "SECOND_CUSTOM": "(?:(?:[0-5]?[0-9]|60))", + "LOGDATE": "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "HOST": "%{HOSTNAME}|%{IP}", + "ACCOUNT": "((\"(%{DATA}|)\" <%{WORD}>)|(%{WORD}))" + }, + "ignore_failure" : true + } +}, + { + "convert" : { + "field" : "cdr.hour", + "type": "integer" + } + }, + { + "convert" : { + "field" : "cdr.minute", + "type": "integer" + } + }, + { + "convert" : { + "field" : "cdr.second", + "type": "integer" + } + }, + { + "script": { + "source": "ctx.cdr.call_duration = ctx.cdr.hour * 3600 + ctx.cdr.minute * 60 + ctx.cdr.second" + } + }, + { + "remove" : { + "field" : "cdr.hour" + } + }, + { + "remove" : { + "field" : "cdr.minute" + } + }, + { + "remove" : { + "field" : "cdr.second" + } + } + ], + "on_failure" : [ + { + "set" : { + "field" : "error", + "value" : "{{_ingest.on_failure_message}}" + } + } + ] +} \ No newline at end of file diff --git a/pipelines/fail2ban.json b/pipelines/fail2ban.json new file mode 100644 index 0000000..81a7026 --- /dev/null +++ b/pipelines/fail2ban.json @@ -0,0 +1,21 @@ +{ + "processors": [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:f2b.log_datetime} %{F2BSERVICE} (\\s.*)\\[%{POSINT:f2b.pid}\\]: %{SEVERITY:f2b.severity} (\\s.*)\\[%{DATA:f2b.iptables_chain}\\] %{F2BMESSAGE}" + ], + "pattern_definitions" : { + "REASON" : "(?:.+)", + "SEVERITY" : "(?:.+)", + "F2BSERVICE" : "%{WORD}.%{WORD:f2b.service}", + "F2BACTION" : "%{GREEDYDATA:f2b.action}", + "F2BMESSAGE" : "(%{F2BACTION} %{IP:f2b.remote_ip})|(%{IP:f2b.remote_ip} %{F2BACTION})", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}" + }, + "ignore_failure" : true + } + } + ] +} \ No newline at end of file diff --git a/pipelines/mailboxlog.json b/pipelines/mailboxlog.json new file mode 100644 index 0000000..5c5c050 --- /dev/null +++ b/pipelines/mailboxlog.json @@ -0,0 +1,189 @@ +{ + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] mailop - (%{MOVINGMESSAGE}|%{ADDINGMESSAGE}|%{DELETEMESSAGE})" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}(|-%{POSINT:mail.pid})(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(||ip=%{IP:mail.ip};)", + "ADDINGMESSAGE" : "%{DATA:mail.command}: id=%{NONNEGINT:mail.id}, Message-ID=<%{DATA:mail.message_id}>, parentId=%{DATA}, folderId=%{NONNEGINT:mail.folder_id}, folderName=%{DATA:mail.folder_name} acct=%{NUMBER:mail.acct}.", + "MOVINGMESSAGE" : "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\) to Folder %{DATA:mail.folder_name} \\(id=%{NONNEGINT:mail.folder_id}\\)", + "DELETEMESSAGE" : "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\).", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] nginxlookup - %{DATA:mail.reason}:%{GREEDYDATA:mail.client_name}" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok": { + "field": "message", + "patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4}|%{USERDATA5}|%{USERDATA6})\\] %{WORD} - %{COMMAND:mail.command} %{GREEDYDATA:mail.reason}"], + "pattern_definitions" : { + "MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})", + "ACCOUNT": "(%{WORD}|%{EMAIL})", + "USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA2": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA3": "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA4": "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};", + "USERDATA5": "(name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};", "USERDATA6": "(name=%{ACCOUNT:mail.client_name};|)oip=%{IP:mail.remote_ip};oport=%{POSINT:mail.port};oproto=%{WORD:mail.proto};soapId=%{DATA:mail.soap_id};(mid=%{POSINT};ds=%{WORD}};|)", + "COMMAND": "%{WORD}(| %{WORD})", + "LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] index - %{DATA:mail.reason} \\{%{GREEDYDATA:mail.message}\\}" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAILADDRESS})", + "USERDATA1" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}" + }, + "ignore_failure" : true + } + }, + { + "grok": { + "field": "message", + "patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2})\\] nginxlookup - %{DATA:mail.reason}:%{GREEDYDATA:mail.client_name}"], + "pattern_definitions" : { + "MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT": "(%{WORD}|%{EMAIL})", + "USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA2": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};ip=%{IP:mail.ip};", + "LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok": { + "field": "message", + "patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4}|%{USERDATA5}|%{USERDATA6}|%{USERDATA7})\\] %{WORD} - %{COMMAND:mail.command} %{GREEDYDATA:mail.reason}"], + "pattern_definitions" : { + "MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})", + "ACCOUNT": "(%{WORD}|%{EMAIL})", + "USERDATA1": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(ip=%{IP:mail.ip};|)oip=%{IP:mail.remote_ip};(via=%{DATA:mail.via};|)ua=%{DATA:mail.ua};(cid=%{POSINT:mail.cid};|)", + "USERDATA2": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};(via=%{DATA:mail.via};|)(ua=%{DATA:mail.ua};|)cid=%{POSINT:mail.cid};", "USERDATA3": "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA4": "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};", + "USERDATA5": "(name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.mail.client_account};|)(mid=%{POSINT:mail.mid};|)ip=%{IP:mail.ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};", + "USERDATA7": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};(port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA6": "(name=%{ACCOUNT:mail.client_name};|)oip=%{IP:mail.remote_ip};oport=%{POSINT:mail.port};oproto=%{WORD:mail.proto};soapId=%{DATA:mail.soap_id};(mid=%{POSINT};ds=%{WORD}};|)", + "COMMAND": "%{WORD}(| %{WORD})", + "LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + + }, + { + "grok": { + "field": "message", + "patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(|%{USERDATA1}|%{USERDATA2}|%{USERDATA3})\\] mailop - (%{MOVINGMESSAGE}|%{ADDINGMESSAGE}|%{DELETEMESSAGE})"], + "pattern_definitions" : { + "MAILSERVICE": "%{DATA:mail.service}(|-%{POSINT:mail.pid})(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT": "(%{WORD}|%{EMAIL})", + "USERDATA1": "name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};port=%{POSINT:mail.port};ua=%{DATA:mail.ua};soapId=%{DATA:mail.soap_id};", + "USERDATA2": "name=%{ACCOUNT:mail.client_name};mid=%{POSINT:mail.mid};(||ip=%{IP:mail.ip};)", + "USERDATA3": "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};", + "ADDINGMESSAGE": "%{DATA:mail.command}: id=%{NONNEGINT:mail.id}, Message-ID=<%{DATA:mail.message_id}>, parentId=%{DATA}, folderId=%{NONNEGINT:mail.folder_id}, folderName=%{DATA:mail.folder_name} acct=%{GREEDYDATA:mail.acct}.", + "MOVINGMESSAGE": "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\) to Folder %{DATA:mail.folder_name} \\(id=%{NONNEGINT:mail.folder_id}\\)", + "DELETEMESSAGE": "%{DATA:mail.command} \\(id=%{NONNEGINT:mail.id}\\).", + "LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok": { + "field": "message", + "patterns": ["%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA5}|%{USERDATA7})\\] %{COMMAND:mail.command} - %{UPLOAD_COMMAND:mail.commands} %{GREEDYDATA:mail.reason}"], + "pattern_definitions" : { + "MAILSERVICE": "%{DATA:mail.service}-%{POSINT:mail.pid}(|:(%{DATA:mail.http_method}:|)%{URI:mail.http_request_url})", + "ACCOUNT": "(%{WORD}|%{EMAIL})", + "USERDATA5": "(name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)(mid=%{POSINT:mail.mid};|)ip=%{IP:mail.ip};port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(via=%{DATA:mail.via};|)soapId=%{DATA:mail.soap_id};", + "USERDATA7": "name=%{ACCOUNT:mail.client_name};(aname=%{ACCOUNT:mail.client_account};|)mid=%{POSINT:mail.mid};oip=%{IP:mail.remote_ip};(port=%{POSINT:mail.port};|)ua=%{DATA:mail.ua};(soapId=%{DATA:mail.soap_id};|)", + "COMMAND": "%{WORD}", + "LOGDATE": "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "UPLOAD_COMMAND": "(Received plain: Upload:|saveUpload\\(\\): received Upload:)", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - %{REASON:mail.reason}" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};", + "REASON" : "Account is lockout, %{GREEDYDATA}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - cmd=%{DATA:mail.command}; account=%{ACCOUNT:mail.client_account}; protocol=%{WORD:mail.protocol}; error=%{DATA:mail.error} \\[%{EMAIL:mail.client_name}\\], %{REASON:mail.reason};" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "REASON" : "%{DATA}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + } + ] +} \ No newline at end of file diff --git a/pipelines/maillog-with-geoip.json b/pipelines/maillog-with-geoip.json new file mode 100644 index 0000000..74a3dcd --- /dev/null +++ b/pipelines/maillog-with-geoip.json @@ -0,0 +1,41 @@ +{ + "processors": [ + { + "pipeline" : { + "name": "maillog" + } + }, + { + "pipeline" : { + "name": "amavis" + } + }, + { + "pipeline" : { + "name": "mailboxlog" + } + }, + { + "pipeline" : { + "name": "zimbralog" + } + }, + { + "pipeline" : { + "name": "fail2ban" + } + }, + { + "geoip" : { + "field" : "mail.remote_ip", + "ignore_missing" : true + } + }, + { + "geoip" : { + "field" : "f2b.remote_ip", + "ignore_missing" : true + } + } + ] +} \ No newline at end of file diff --git a/pipelines/maillog.json b/pipelines/maillog.json new file mode 100644 index 0000000..fb8e814 --- /dev/null +++ b/pipelines/maillog.json @@ -0,0 +1,774 @@ +{ + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{MAILSERVICE:mail.service}-%{NONNEGINT:mail.pid}\\] \\[name=%{EMAIL:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.mail.remote_ip};via=%{IP:mail.relay_ip}\\(%{DATA}\\);%{DATA}\\] %{DATA} - %{REASON:mail.reason}" + ], + "pattern_definitions" : { + "RELAYPORT" : "[0-9]+", + "REASON" : "(?:.+)", + "SEVERITY" : "(?:.+)", + "CONNECTIONSTATUS" : "connect|disconnect", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "(?:.+)", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ID" : "(?:[0-9]+)", + "CLIENTDATA" : "name=%{EMAIL:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "REMOTEPORT" : "[0-9]+", + "SOAPID" : "soapId=%{DATA:mail.soap_id}", + "PROTOCOL" : "%{WORD}", + "SEVERITY" : "(?:.+)", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "ID" : "(?:[0-9]+)", + "SOAPID" : "soapId=%{WORD:mail.soap_id}", + "CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};ua=%{DATA:mail.ua};%{SOAPID};", + "REMOTEPORT" : "[0-9]+", + "PROTOCOL" : "%{WORD}", + "SEVERITY" : "(?:.+)", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "ID" : "(?:[0-9]+)", + "CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};", + "REMOTEPORT" : "[0-9]+", + "SOAPID" : "soapId=%{DATA:mail.soap_id}", + "SEVERITY" : "(?:.+)", + "PROTOCOL" : "%{WORD}", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "ID" : "(?:[0-9]+)", + "CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};ua=%{DATA:mail.ua};%{SOAPID};", + "REMOTEPORT" : "[0-9]+", + "SOAPID" : "soapId=%{WORD:mail.soap_id}", + "PROTOCOL" : "%{WORD}", + "SEVERITY" : "(?:.+)", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol}; %{ERRORMESSAGE};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "ID" : "(?:[0-9]+)", + "CLIENTDATA" : "name=%{ACCOUNT:mail.client_name};oip=%{IP:mail.remote_ip};oport=%{REMOTEPORT:mail.remote_port};oproto=%{PROTOCOL:mail.protocol};%{SOAPID};", + "REMOTEPORT" : "[0-9]+", + "SOAPID" : "soapId=%{DATA:mail.soap_id}", + "PROTOCOL" : "%{WORD}", + "SEVERITY" : "(?:.+)", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "ERRORMESSAGE" : "error=%{DATA:mail.error}( \\[%{ACCOUNT}\\]|), %{DATA:mail.reason}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{SEVERITY:mail.severity} \\[%{URLDATA}\\] \\[%{CLIENTDATA}\\] %{COMMAND}; account=%{ACCOUNT:mail.client_account}; protocol=%{DATA:mail.auth_protocol};" + ], + "pattern_definitions" : { + "URLDATA" : "%{DATA:mail.qtp}-%{ID:mail.id}:(%{DATA}:%{URI:mail.zimbra_url}|%{URI:mail.zimbra_url})", + "ID" : "(?:[0-9]+)", + "CLIENTDATA" : "name=%{EMAIL:mail.client_name};oip=%{IP:mail.remote_ip};ua=%{DATA:mail.ua};%{SOAPID};", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "REMOTEPORT" : "[0-9]+", + "SOAPID" : "soapId=%{DATA:mail.soap_id}", + "PROTOCOL" : "%{WORD}", + "SEVERITY" : "(?:.+)", + "COMMAND" : "%{DATA} cmd=%{WORD:mail.command}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{WORD:mail.severity}(\\s.+)\\[%{MAILSERVICE}\\] \\[(%{USERDATA1}|%{USERDATA2}|%{USERDATA3}|%{USERDATA4})\\] security - cmd=%{COMMAND:mail.command}; account=%{ACCOUNT:mail.client_name}; protocol=%{WORD:mail.protocol}" + ], + "pattern_definitions" : { + "MAILSERVICE" : "%{DATA:mail.service}-%{NONNEGINT:mail.pid}(|:%{DATA:mail.http_method}:%{URI:mail.http_request_url})", + "ACCOUNT" : "(%{WORD}|%{EMAIL})", + "USERDATA1" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};cid=%{POSINT:mail.cid};", + "USERDATA2" : "name=%{ACCOUNT:mail.client_name};ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA3" : "ip=%{IP:mail.ip};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via};ua=%{DATA:mail.ua};cid=%{POSINT:mail.cid};", + "USERDATA4" : "ip=%{IP:mail.ip};cid=%{POSINT:mail.cid};oip=%{IP:mail.remote_ip};via=%{DATA:mail.via}\\);ua=%{DATA:mail.ua};", + "COMMAND" : "%{WORD}", + "LOGDATE" : "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: to=<%{DATA:mail.to}>,(?:\\sorig_to=<%{EMAIL:mail.orig_to}>,)? relay=%{RELAY}, delay=%{NUMBER:mail.delay}, delays=%{DELAYS}, dsn=%{DSN:mail.dsn}, status=%{STATUS:mail.status} \\(%{DATA:mail.reason}\\)" + ], + "pattern_definitions" : { + "RELAY" : "(?:%{HOSTNAME:mail.relay_host}(?:\\[%{IP:mail.relay_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}", + "PERMERROR" : "5[0-9]{2}", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "STATUS" : "sent|deferred|bounced|expired", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{POSTFIXACTION:mail.postfix_action}: %{DATA:mail.command} from %{REMOTE}: %{PERMERROR:mail.response_code} %{DSN:mail.dsn} %{DATA}: %{DATA:mail.reason}; (from=<%{EMAIL:mail.from}> |)to=<%{EMAIL:mail.to}> proto=%{DATA:mail.proto} helo=<%{HELO}>" + ], + "pattern_definitions" : { + "POSTFIXACTION" : "discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "PERMERROR" : "(4|5)[0-9][0-9]", + "DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "HELO" : "(?:\\[%{IP:mail.helo}\\]|%{HOSTNAME:mail.helo}|%{DATA:mail.helo})", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{MESSAGELEVEL:mail.message_level}: hostname %{HOSTNAME:mail.remote_host} %{DATA:\n } address %{IP:mail.remote_ip}" + ], + "pattern_definitions" : { + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "MESSAGELEVEL" : "reject|warning|error|fatal|panic", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} %{DIRECTION:mail.connect_direction} %{REMOTE}:%{REMOTEPORT:mail.remote_port}: %{REASON:mail.reason}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "REMOTEPORT" : "[0-9]+", + "REASON" : "(?:.+)", + "CONNECTIONSTATUS" : "connect|disconnect", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "DIRECTION" : "(to|from)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} %{DIRECTION:mail.connect_direction} %{REMOTE}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "REMOTEPORT" : "[0-9]+", + "REASON" : "(?:.+)", + "CONNECTIONSTATUS" : "connect|disconnect", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "DIRECTION" : "(to|from)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} from \\[%{IPORHOST:mail.remote_host}\\]:%{PORT:mail.remote_port} to \\[%{IPORHOST:mail.local_host}\\]:%{PORT:mail.local_port}" + ], + "pattern_definitions" : { + "PORT" : "[0-9]+", + "CONNECTIONSTATUS" : "CONNECT|DISCONNECT|connect|disconnect", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTIONSTATUS:mail.connection_status} after %{DATA:mail.command} from %{REMOTE}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "CONNECTIONSTATUS" : "lost connection", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{REMOTE}, sasl_method=%{DATA:mail.sasl_method}, sasl_username=%{EMAIL:mail.username}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: message-id=(|<)%{DATA:mail.message_id}(|>)" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: from=<%{EMAIL:mail.from}>, size=%{DATA:mail.size}, nrcpt=%{DATA:mail.nrcpt} \\(%{DATA:mail.reason}\\)" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{MESSAGESTATUS:mail.reason}" + ], + "pattern_definitions" : { + "MESSAGESTATUS" : "removed", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: (%{HOSTDATA}:\\s|\\s)%{DATA:mail.error}: %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "HOSTDATA" : "%{HOSTNAME:mail.remote_host}\\[%{IPORHOST:mail.remote_ip}\\]", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "SEVERITY" : "(warning|info|error)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{DATA:mail.reason} \\[%{IP:mail.remote_ip}\\]:%{NONNEGINT:mail.remote_port}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "EMAIL" : "([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{DATA:mail.reason} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]: TLS%{DATA:mail.tls_proto} \\(%{DATA}\\)" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}(|:) %{CONNECTIONSTATUS:mail.connection_status} from %{REMOTE} ehlo=%{NONNEGINT:mail.ehlo} mail=%{DATA} rcpt=%{NONNEGINT:mail.rcpt} data=%{WORD:mail.data} (noop=%{NONNEGINT:mail.noop} |)quit=%{NONNEGINT:mail.quit} commands=%{NONNEGINT:mail.commands}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "CONNECTIONSTATUS" : "connect|disconnect", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: hostname %{HOSTNAME:mail.remote_host} %{DATA:mail.reason} address %{IP:mail.remote_ip}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{NONNEGINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "SEVERITY" : "(warning|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{SASL:mail.sasl_message}: %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "SASL" : "SASL %{DATA}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "SEVERITY" : "(warning|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{POSTFIXACTION:mail.postfix_action}: %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "POSTFIXACTION" : "statistics", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{POSTFIXACTION:mail.postfix_action} %{GREEDYDATA:mail.reason}: retained=%{NONNEGINT:mail_cache_retained} dropped=%{NONNEGINT:mail.cache_dropped} entries" + ], + "pattern_definitions" : { + "POSTFIXACTION" : "cache", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{MSG:mail.reason}: %{QUEUEID:mail.qid2}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "MSG" : "(sender (non-delivery|delivery status) notification)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: resent-message-id=<%{DATA:mail.message_id}>" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "MSG" : "sender non-delivery notification" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: host %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] said: %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: conversation with %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: from=<%{EMAIL:mail.from}>, status=%{WORD:mail.status}, %{GREEDYDATA:mail.reason}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: host %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\] %{STATUS:mail.status} %{DATA}: %{DATA:mail.reason};" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "STATUS" : "refused", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}(:|) %{QUEUEID:mail.qid}: %{POSTFIXACTION:mail.postfix_action}: %{DATA:mail.command} from %{REMOTE}: <%{EMAIL:mail.client}>: %{DATA:mail.reason}; from=<%{EMAIL:mail.from}> to=<%{EMAIL:mail.to}> proto=%{DATA:mail.proto} helo=<%{HELO}>" + ], + "pattern_definitions" : { + "POSTFIXACTION" : "discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "HELO" : "(?:\\[%{IP:mail.helo}\\]|%{HOSTNAME:mail.helo}|%{DATA:mail.helo})", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(|([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME})", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE} %{QUEUEID:mail.qid}: client=%{REMOTE}" + ], + "pattern_definitions" : { + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{STATUS:mail.connection_status}: %{DATA:mail.reason} from %{REMOTE}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "STATUS" : "reject", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "SEVERITY" : "(warning|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{REASON:mail.reason} after %{DATA:mail.command} from %{REMOTE}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "REASON" : "((too many errors)|(timeout))", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} from %{REMOTE} in %{DATA:mail.command} command: %{GREEDYDATA:mail.message}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "STATUS" : "reject", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)", + "SEVERITY" : "(warning|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUEID:mail.qid}: %{DATA:mail.reason} for %{REMOTE}$" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "STATUS" : "reject", + "REMOTE" : "(?:%{HOSTNAME:mail.remote_host}(?:\\[%{IP:mail.remote_ip}\\](?::[0-9]+(.[0-9]+)?)?)?)", + "HOSTNAME" : "\b(?:[0-9A-Za-z][0-9A-Za-z\\-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\b)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} for %{HOSTNAME:mail.remote_host}: %{IP:mail.remote_ip}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "SEVERITY" : "(warning|error|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{SEVERITY:mail.severity}: %{DATA:mail.reason} (from|with) %{HOSTDATA}%{GREEDYDATA:mail.message}" + ], + "pattern_definitions" : { + "QUEUEID" : "(?:[A-F0-9]+|NOQUEUE)", + "HOSTDATA" : "%{HOSTNAME:mail.remote_host}\\[%{IPORHOST:mail.remote_ip}\\]", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY}(\\s+)%{TIME}", + "MAILSERVICE" : "%{WORD}/%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "SEVERITY" : "(warning|info)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{IP:mail.remote_ip}:%{POSINT:mail.remote_port} -%{REMOTEUSER}- \\[%{LOGDATETIME:mail.log_datetime} %{ISO8601_TIMEZONE}\\](\\s.+?|)\\\"%{REQUEST}\\ %{POSINT:mail.http_status} %{NONNEGINT:mail.http_bytes_sent} \\\"%{REFERER:mail.http_referer}\\\" \\\"%{DATA:mail.http_user_agent}\\\" \\\"%{IP:mail.ip1}:%{POSINT:mail.port1}\\\" (\\\"%{IP:mail.ip2}:%{POSINT:mail.port2}\\\")" + ], + "pattern_definitions" : { + "REASON" : "(?:.+)", + "REMOTEUSER" : "(\\s|%{DATA:mail.remote_user})", + "REQUEST" : "%{DATA:mail.http_method} %{URI:mail.http_request_url} %{DATA}", + "REFERER" : "%{URI}|-", + "SEVERITY" : "(?:.+)", + "LOGDATETIME" : "%{MONTHDAY}/%{MONTH}/20%{YEAR}:%{TIME}" + }, + "ignore_failure" : true + } + }, + { + "date_index_name" : { + "field" : "@timestamp", + "index_name_prefix" : "maillog_filebeat-", + "date_rounding" : "d", + "index_name_format" : "yyyy-MM-dd" + } + } + ] + +} diff --git a/pipelines/zimbralog.json b/pipelines/zimbralog.json new file mode 100644 index 0000000..7b245b9 --- /dev/null +++ b/pipelines/zimbralog.json @@ -0,0 +1,89 @@ +{ + "processors": [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{CONNECTION:mail.connection_status} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\](%{DATA:mail.reason}|)$" + ], + "pattern_definitions" : { + "HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "CONNECTION" : "(connect|disconnect)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: client=%{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]" + ], + "pattern_definitions" : { + "HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "QUEUED" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: filter: %{DATA} from %{HOSTNAME:mail.remote_host}\\[%{IP:mail.remote_ip}\\]: %{DATA}; from=<%{EMAIL:mail.from}> to=<%{EMAIL:mail.to}> proto=%{DATA:mail.protocol} helo=<%{HOSTNAME:mail.helo}>" + ], + "pattern_definitions" : { + "HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)", + "PORT" : "(?:[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "QUEUED" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: to=<%{EMAIL:mail.to}>, (|(orig_to=<%{EMAIL:mail.orig_to}>, ))((relay=%{HOSTNAME:mail.relay_host}\\[%{IP:mail.relay_ip}\\]:%{PORT:mail.relay_port})|(relay=%{WORD:mail.relay_host})), (conn_use=%{WORD}, |)delay=%{NUMBER:mail.delay}, delays=%{DELAYS}, dsn=%{DSN:mail.dsn}, status=%{WORD:mail.status} \\(%{DATA:mail.reason}\\)((: %{DATA}: queued as %{QUEUED:mail.qid2})|)$" + ], + "pattern_definitions" : { + "HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)", + "PORT" : "(?:[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}", + "DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}", + "QUEUED" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{LOGDATE:mail.log_datetime} %{DATA} %{MAILSERVICE}: %{QUEUED:mail.qid}: from=<%{EMAIL:mail.from}>, size=%{WORD:mail.size}, nrcpt=%{WORD} \\(%{DATA:mail.reason}\\)$" + ], + "pattern_definitions" : { + "HOSTNAME" : "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)", + "PORT" : "(?:[0-9]+)", + "LOGDATE" : "%{MONTH}(\\s+)%{MONTHDAY} %{TIME}", + "MAILSERVICE" : "%{DATA:mail.service}\\[%{POSINT:mail.pid}\\]", + "EMAIL" : "(([?a-zA-Z0-9_.+-=:]+)@%{HOSTNAME}|)", + "DELAYS" : "%{NUMBER:mail.delay_prequeued}/%{NUMBER:mail.delay_queued}/%{NUMBER:mail.delay_connection}/%{NUMBER:mail.delay_sended}", + "DSN" : "%{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}", + "QUEUED" : "(?:[A-F0-9]+|NOQUEUE)" + }, + "ignore_failure" : true + } + } + ] + } \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..368e2ec --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,3 @@ +# Вспомогательные скрипты и утилиты для ElasticSearch + +- elastic_idex_del.pl - удаление индексов ES старше определоенной даты \ No newline at end of file diff --git a/scripts/elastic_index_del.pl b/scripts/elastic_index_del.pl new file mode 100755 index 0000000..c94d6af --- /dev/null +++ b/scripts/elastic_index_del.pl @@ -0,0 +1,80 @@ +#!/usr/bin/perl +######################################### +# +# ElasticSearch index remover +# +# Author: Sergey Kalinin +# +# https://nuk-svk.ru +# svk@nuk-svk.ru +######################################### + +use Getopt::Long; + +use vars qw(%opts); +use feature qw(say); + + +my $help = 0; +my $debug = 0; +my $domain_file; + +GetOptions(\%opts, 'host=s', 'days=s', 'data-type=s', 'help', 'verbose'); + +if (defined($opts{'help'})) +{ + print STDERR < + + 5.0 + 2020-12-30T12:08:33Z + + + Elastic cluster + + + + + +