commit 4113bcbd7d52b7b714226772aafc4b332e9112ce Author: svkalinin Date: Fri Jun 14 16:33:02 2024 +0300 Восстановление репозитория diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b8ffd93 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:alpine3.16 AS build +RUN apk --no-cache add gcc g++ make git +WORKDIR /go/src/app +COPY . . +RUN go get net/netip +RUN go get ./... + +RUN GOOS=linux go build -ldflags="-s -w" -o ./bin/zimbra-alarm ./zimbra-alarm.go +FROM alpine:3.16 +RUN apk add tzdata +WORKDIR /usr/bin +COPY --from=build /go/src/app/bin /go/bin + +COPY entrypoint.sh . + +# COPY cronjobs /etc/crontabs/root + +# start crond with log level 8 in foreground, output to stderr +# CMD ["crond", "-f", "-d", "8"] + +ENTRYPOINT ["/bin/sh", "./entrypoint.sh"] + diff --git a/README.md b/README.md new file mode 100644 index 0000000..4c46a2b --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# Описание + +Получение и обработка данных о пользовательских сессиях (авторизация) при соединении с почтовой системой. И блокирование нежелательных адресов. + +Для работы требуется ElasticSearch, PostgreSQL, докер (опционально). + +## zimbra-alarm + +Программа zimbra-alarm получает из elasticsearch, список пользовательских сессий за заданный промежуток времени (4 минуты). Для определения из каких индексов производить выборку данных, первоначально загружается список всех индексов из эластика (без временной метки), далее выбираются имена индексов соответствующие заданным шаблонам (префиксам), по умолчанию это maillog-mailbox. И для каждого типа индексов (например maillog_mailbox1_filebeat*, maillog_mailbox2_filebeat* и т.п.) запускается параллельный процесс поиска и выгрузки данных. Данные обрабатываются и складываются в таблицу 'mail_sessions' СУБД Postgresql, для каждого почтового аккаунта соединения с одного ip адреса суммируются в одну запись с зафиксированной меткой времени. +Почтовые аккаунты сохраняются в таблице mail_accounts. + +После этого запускается процесс анализа данных о сессиях и генерации сообщений о "тревогах" (таблица 'alarm'): + +1. Если количество сессий с одного адреса в один аккаунт превышает заданное число (50), будет создано сообщение об аномалии с типом "2" ("Количество сессий больше заданного") +2. Если код geoip ip-адреса не равен "RU" - будет создана запись c ip-адресом и типом "1" ("Неразрешенный код страны") +3. Если зафиксированы соединения в один почтовый аккаунт с разных адресов, то будет сгенерировано сообщение с типом "3" ("Вход в аккаунт более чем с одного IP") +4. Если с одного ip-адреса, для которого есть запись с типом 1 (неразрешенный код страны), были попытки входа в несколько учетных записей, то будет добавлена запись с типом "4" ("Вход более чем в один аккаунт") + +Если для одного почтового аккаунта или одного ip-адреса будет сгенерировано сообщение о тревоги с одним и тмеже типом, то существующая запись о тревоге обновиться с последней меткой времени. + +Программу можно запускать как в консоли так и в docker-контейнере. + +## alarmaction + +Программа "alarmaction" предназначена для выполнения действий по "тревогам". На данный момент доступно блокирование ip адресов через "iptables". Устанавливается на целевых узлах (в нашем случае MTA сервера mail и mail2). Запуск производится через службу "crond". + +При запуске программы, происходит чтение таблицы "alarm" из БД. Выбираются все записи с типом 4 (alarm_type=4) и пустым полем "_action_timestamp, где "" - укороченное имя узла (в нашем случае "mail_action_timestamp" и "mail2_action_timestamp"). + +Выбранные ip-адреса будут блокированы через iptables на каждом из узлов и после успешного выполнения блокровки в соответствующая запись в таблице будет изменена (добавлена временная метка в нужное поле, см. выше). + +Все адреса в iptables заносятся в цепочку "auto_blocked" которая будет создана при первом запуске alarmaction. + +Для исключения блокирования определенных адресов или подсетей в таблицу filter необходимо добавить соответствующую запись (адрес и маску): + +``` +INSERT INTO filter(remote_ip, description) values('192.168.0.0/16', 'Локальная сеть') +``` + +# Запуск + +zimbra-alarm можно запускать как по времени так и в докер-контейнере. alarmaction - запускается по времени. + +Для работы обоих программ нужно установить переменные окружения: + +zimbra-alarm: + + - ELASTICSEARCH_URL=https://${ELASTIC_USER}:${ELASTIC_PASSWORD}@${ELASTIC_HOST}:9200 - строка соедения с ElasticSearch + - PGHOST=dbserver - сервер СУБД (PostgreSQL) + - PGDATABASE=mail_alarm - имя базы данных + - PGUSER=mail_alarm - пользователь БД + - PGPASSWORD={PGPASSWORD} - пароль пользователя + - REQUEST_TIME_RANGE=4 - время за которое запрашиваются данные в минутах (по умолчанию 4м) + - REQUEST_ROWS=10000 - количество строк возвращаемое запросом (по умолчанию 10000) + +команда запуска: + +см. entrypoint.sh + +``` +zimbra-alarm --indexname "maillog_mailbox" -timerange "${REQUEST_TIME_RANGE}m" -operation es-request-data -pg-insert -request-rows ${REQUEST_ROWS} +``` + +alarmaction: + + - PGHOST=dbserver + - PGUSER=pguser + - PGDATABASE=maildb + - PGPASSWORD=pgpassword + - ALARM_LOG_FILE=/var/log/alarmaction.log + +команда запуска: + +см. alarmaction.cron + +``` +alarmaction +``` diff --git a/alarmaction-v1.go b/alarmaction-v1.go new file mode 100644 index 0000000..cc1d2e1 --- /dev/null +++ b/alarmaction-v1.go @@ -0,0 +1,320 @@ +//---------------------------------------------------------------------------- +// Получение и обработка данных о пользовательских сессиях (авторизация) +// при соединении с почтовой системой. И блокирование нежелательных адресов. +// --------------------------------------------------------------------------- +// Версия 1 (iptables) +// --------------------------------------------------------------------------- + + +package main + +import ( + "log" + "strings" + "net/netip" + "time" + "context" + "fmt" + "os" + "io/ioutil" + "path/filepath" + + "github.com/coreos/go-iptables/iptables" + "github.com/jackc/pgx/v5" + // "github.com/jackc/pgtype" +) + +type Alarm struct { + Id int32 + Mail_account_id int32 + Remote_ip netip.Prefix + Country_code string + Last_time string + Last_check_timestamp time.Time + Action_timestamp time.Time + Alarm_type int + Alarm_action bool + Hostname string +} + +// PIDFile stored the process id +type PIDFile struct { + path string +} + +var PID_file_path = "/var/run/alarmaction.pid" + +// Remove the pid file +// func (file PIDFile) PIDRemove() error { + // return os.Remove(file.path) +// } + +func ProcessExit(exit_code int) { + if exit_code == 0 { + os.Remove(PID_file_path) + } + os.Exit(exit_code) +} + +// Выборка записей по конкретному IP +// запросы разные в зависимости от типа тревоги +func PgSelectAlarmRemoteIP(alarm Alarm) pgx.Rows { + var query string + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + switch alarm.Alarm_type { + case 4: + query = fmt.Sprintf("select id, remote_ip from alarm where alarm_type = %v and \"%v_action_timestamp\" is NULL", alarm.Alarm_type, alarm.Hostname) + } + log.Println(query) + rows, err_select := conn.Query(context.Background(), query) + + if err_select != nil { + log.Println("Query alarms failed PgSelectAlarmRemoteIP:", err_select) + ProcessExit(1) + } + if len(rows.FieldDescriptions()) == 0 { + ProcessExit(1) + } + + // log.Println("Selected an:",len(rows.FieldDescriptions()), "records") + + defer conn.Close(context.Background()) + return rows +} +// Проверка на соответствие адреса записи в filter +func PgSelectFilterIP(ip netip.Prefix) int32 { + var query string + var id int32 + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + // проверяем адрес на совпадение с фильтром (для исключения) + // select * from filter where remote_ip >>= inet '209.85.161.20/32' + query = fmt.Sprintf("select id from filter where remote_ip >>= inet '%v'", ip) + log.Println(query) + err_select := conn.QueryRow(context.Background(), query).Scan(&id) + + if err_select != nil { + log.Println("Select filter for remote ip", ip, "error:", err) + if err_select.Error() == "no rows in result set" { + id = 0 + } else { + ProcessExit(1) + } + } + return id +} +func PgUpdateAlarm(conn *pgx.Conn, alarm Alarm, id int32) int32 { + log.Println("Update alarm data", alarm) + t := time.Now().Format("2006-01-02 15:04:05") + query := fmt.Sprintf("update alarm set \"%v_action_timestamp\"='%v' where id=%v returning id", alarm.Hostname, t, id) + log.Println("Update alarm query:", query) + err := conn.QueryRow(context.Background(), query).Scan(&id) + if err != nil { + log.Printf("Update alarm data failed: %v\n", err) + } + + return id +} + +func IpTablesCreateChain (sChain string) bool { + ipt, err := iptables.New() + if err != nil { + log.Printf("Failed to new up an IPtables intance. ERROR: %v", err) + ProcessExit(1) + } + var chain_exists bool + chain_exists, err = ipt.ChainExists("filter", sChain) + if err != nil { + log.Printf("Failed to checking chain (%v). ERROR: %v", sChain , err) + return false + } + if chain_exists { + return true + } else { + err = ipt.NewChain("filter", sChain) + if err != nil { + log.Printf("Failed to creating chain (%v). ERROR: %v", sChain , err) + return false + } + err = ipt.AppendUnique("filter", "INPUT", "-j", sChain) + if err != nil { + log.Printf("Failed to append chain. ERROR: %v", err) + return false + } + } + + return true + +} + +func BlockIP(ip netip.Prefix, sChain string, dChain string) bool { + // Some default chain names + + // Get a new iptables interface + ipt, err := iptables.New() + if err != nil { + log.Printf("Failed to new up an IPtables intance. ERROR: %v", err) + ProcessExit(1) + } + // Build out the ipstring(add /32 to the end) + ipstr := fmt.Sprintf("%s", ip) + res, _ := ipt.Exists("filter", sChain, "-s", ipstr, "-j", dChain) + if !res { + // Use the appendUnique method to put this in iptables, but only once + err = ipt.AppendUnique("filter", sChain, "-s", ipstr, "-j", dChain) + if err != nil { + log.Printf("Failed to ban an ip(%v). ERROR: %v", ipstr , err) + return false + } + } else { + log.Printf("Failed to ban an ip(%v). ERROR: Address already banned", ipstr) + } + + // Since we made it here, we won + return true +} + +// just suit for linux +func processExists(pid string) bool { + if _, err := os.Stat(filepath.Join("/proc", pid)); err == nil { + return true + } + return false +} + +func checkPIDFILEAlreadyExists(path string) error { + if pidByte, err := ioutil.ReadFile(path); err == nil { + pid := strings.TrimSpace(string(pidByte)) + if processExists(pid) { + return fmt.Errorf("ensure the process:%s is not running pid file:%s", pid, path) + } + } + return nil +} + +// NewPIDFile create the pid file +// path specified under production pidfile, file content pid +func NewPIDFile(path string) (*PIDFile, error) { + if err := checkPIDFILEAlreadyExists(path); err != nil { + return nil, err + } + + if err := os.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil { + return nil, err + } + if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil { + return nil, err + } + return &PIDFile{path: path}, nil +} + +func main() { + // Создадим и откроем на запись файл для логов + var alarm_log_file string + if os.Getenv("ALARM_LOG_FILE") == "" { + alarm_log_file = "alarmaction.log" + } else { + alarm_log_file = os.Getenv("ALARM_LOG_FILE") + } + + f, err := os.OpenFile(alarm_log_file, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + + log.SetOutput(f) + + // Создание PID файла + _, err = NewPIDFile(PID_file_path) + if err != nil { + log.Println("error to create the pid file failed:", err.Error()) + ProcessExit(1) + } + + // Проверка переменных окружения + if os.Getenv("PGHOST") == "" { + fmt.Println("Send error: make sure environment variables `PGHOST` was set") + ProcessExit(1) + } + if os.Getenv("PGUSER") == "" { + fmt.Println("Send error: make sure environment variables `PGUSER` was set") + ProcessExit(1) + } + if os.Getenv("PGPASSWORD") == "" { + fmt.Println("Send error: make sure environment variables `PGPASSWORD` was set") + ProcessExit(1) + } + if os.Getenv("PGDATABASE") == "" { + fmt.Println("Send error: make sure environment variables `PGDATABASE` was set") + ProcessExit(1) + } + // if os.Getenv("HOSTNAME") == "" { + // fmt.Println("Send error: make sure environment variables `HOSTNAME` was set") + // ProcessExit(1) + // } + + hostname, err := os.Hostname() + if err != nil { + fmt.Println(err) + ProcessExit(1) + } + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + sChain := "auto_blocked" + dChain := "DROP" + + if !IpTablesCreateChain(sChain) { + log.Println("Iptables chain", sChain, "does not exists") + ProcessExit(1) + } + + var alarm Alarm + + alarm.Hostname = strings.Split(hostname, ".")[0] + alarm.Action_timestamp = time.Unix(time.Now().Unix(), 0) + alarm.Alarm_type = 4 + + res := PgSelectAlarmRemoteIP(alarm) + var id int32 + var ip netip.Prefix + var filter_id int32 + for res.Next() { + values, _ := res.Values() + id = values[0].(int32) + ip = values[1].(netip.Prefix) + filter_id = PgSelectFilterIP(ip) + log.Println(filter_id) + if filter_id > 0 { + log.Println("The ip address", ip, "is conained in filter table") + log.Println(PgUpdateAlarm(conn, alarm, id)) + // continue + } else { + fmt.Println(id, ip) + if BlockIP(ip, sChain, dChain) { + log.Println("The ip address", ip, "was blocked") + alarm.Alarm_action = true + log.Println(PgUpdateAlarm(conn, alarm, id)) + } + } + } + ProcessExit(0) +} diff --git a/alarmaction.cron b/alarmaction.cron new file mode 100644 index 0000000..f1b0f7f --- /dev/null +++ b/alarmaction.cron @@ -0,0 +1,9 @@ +PGHOST=build1.corp.samsonopt.ru +PGUSER=pguser +PGDATABASE=maildb +PGPASSWORD=pgpassword +PATH=$PATH:/usr/sbin +ALARM_LOG_FILE=/var/log/alarmaction.log + +*/1 * * * * root /usr/local/bin/alarmaction >> /var/log/alarmaction.log 2>&1 + diff --git a/alarmaction.go b/alarmaction.go new file mode 100644 index 0000000..09beef8 --- /dev/null +++ b/alarmaction.go @@ -0,0 +1,371 @@ +//---------------------------------------------------------------------------- +// Получение и обработка данных о пользовательских сессиях (авторизация) +// при соединении с почтовой системой. И блокирование нежелательных адресов. +// --------------------------------------------------------------------------- +// Версия 2 (ipset) +// --------------------------------------------------------------------------- + +package main + +import ( + "log" + "strings" + "net/netip" + "time" + "context" + "fmt" + "os" + "io/ioutil" + "path/filepath" + + // "github.com/coreos/go-iptables/iptables" + "github.com/gonetx/ipset" + "github.com/jackc/pgx/v5" + // "github.com/jackc/pgtype" +) + +// проверяем наличие ipset системе +func CheckIPSet() { + if err := ipset.Check(); err != nil { + panic(err) + } +} + +type Alarm struct { + Id int32 + Mail_account_id int32 + Remote_ip netip.Prefix + Country_code string + Last_time string + Last_check_timestamp time.Time + Action_timestamp time.Time + Alarm_type int + Alarm_action bool + Hostname string +} + +// PIDFile stored the process id +type PIDFile struct { + path string +} + +var PID_file_path = "/var/run/alarmaction.pid" + +// Remove the pid file +// func (file PIDFile) PIDRemove() error { + // return os.Remove(file.path) +// } + +func ProcessExit(exit_code int) { + if exit_code == 0 { + os.Remove(PID_file_path) + } + os.Exit(exit_code) +} + +// Выборка записей по конкретному IP +// запросы разные в зависимости от типа тревоги +func PgSelectAlarmRemoteIP(alarm Alarm) pgx.Rows { + var query string + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + switch alarm.Alarm_type { + case 4: + query = fmt.Sprintf("select id, remote_ip from alarm where alarm_type = %v and \"%v_action_timestamp\" is NULL", alarm.Alarm_type, alarm.Hostname) + } + log.Println(query) + rows, err_select := conn.Query(context.Background(), query) + + if err_select != nil { + log.Println("Query alarms failed PgSelectAlarmRemoteIP:", err_select) + ProcessExit(1) + } + if len(rows.FieldDescriptions()) == 0 { + ProcessExit(1) + } + + // log.Println("Selected an:",len(rows.FieldDescriptions()), "records") + + defer conn.Close(context.Background()) + return rows +} +// Проверка на соответствие адреса записи в filter +func PgSelectFilterIP(ip netip.Prefix) int32 { + var query string + var id int32 + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + // проверяем адрес на совпадение с фильтром (для исключения) + // select * from filter where remote_ip >>= inet '209.85.161.20/32' + query = fmt.Sprintf("select id from filter where remote_ip >>= inet '%v'", ip) + log.Println(query) + err_select := conn.QueryRow(context.Background(), query).Scan(&id) + + if err_select != nil { + log.Println("Select filter for remote ip", ip, "error:", err) + if err_select.Error() == "no rows in result set" { + id = 0 + } else { + ProcessExit(1) + } + } + return id +} +func PgUpdateAlarm(conn *pgx.Conn, alarm Alarm, id int32) int32 { + log.Println("Update alarm data", alarm) + t := time.Now().Format("2006-01-02 15:04:05") + query := fmt.Sprintf("update alarm set \"%v_action_timestamp\"='%v' where id=%v returning id", alarm.Hostname, t, id) + log.Println("Update alarm query:", query) + err := conn.QueryRow(context.Background(), query).Scan(&id) + if err != nil { + log.Printf("Update alarm data failed: %v\n", err) + } + + return id +} + +// func IpTablesCreateChain (sChain string) bool { + // ipt, err := iptables.New() + // if err != nil { + // log.Printf("Failed to new up an IPtables intance. ERROR: %v", err) + // ProcessExit(1) + // } + // var chain_exists bool + // chain_exists, err = ipt.ChainExists("filter", sChain) + // if err != nil { + // log.Printf("Failed to checking chain (%v). ERROR: %v", sChain , err) + // return false + // } + // if chain_exists { + // return true + // } else { + // err = ipt.NewChain("filter", sChain) + // if err != nil { + // log.Printf("Failed to creating chain (%v). ERROR: %v", sChain , err) + // return false + // } + // err = ipt.AppendUnique("filter", "INPUT", "-j", sChain) + // if err != nil { + // log.Printf("Failed to append chain. ERROR: %v", err) + // return false + // } + // } +// + // return true + // +// } + +// func BlockIP(ip netip.Prefix, sChain string, dChain string) bool { + // // Some default chain names +// + // // Get a new iptables interface + // ipt, err := iptables.New() + // if err != nil { + // log.Printf("Failed to new up an IPtables intance. ERROR: %v", err) + // ProcessExit(1) + // } + // // Build out the ipstring(add /32 to the end) + // ipstr := fmt.Sprintf("%s", ip) + // res, _ := ipt.Exists("filter", sChain, "-s", ipstr, "-j", dChain) + // if !res { + // // Use the appendUnique method to put this in iptables, but only once + // err = ipt.AppendUnique("filter", sChain, "-s", ipstr, "-j", dChain) + // if err != nil { + // log.Printf("Failed to ban an ip(%v). ERROR: %v", ipstr , err) + // return false + // } + // } else { + // log.Printf("Failed to ban an ip(%v). ERROR: Address already banned", ipstr) + // } +// + // // Since we made it here, we won + // return true +// } +func BlockIP(ip netip.Prefix, sChain string, dChain string) bool { + // create test set even it's exist + // set, err := ipset.New("auto_blocked", ipset.HashIp, ipset.Exist(true), ipset.Timeout(time.Hour)) + ip_address := ip.String() + set, err := ipset.New("auto-blocked-ipset", ipset.HashIp, ipset.Exist(true)) + // output: test + if err != nil { + log.Printf("Failed to create new ipset rule. ERROR: %v", err) + // return false + } else { + log.Printf("New ipset rule %v was created", set.Name()) + } + + // _ = set.Flush() + + // _ = set.Add("1.1.1.1", ipset.Timeout(time.Hour)) + err = set.Add(ip_address) + if err != nil { + log.Printf("Failed to added new %v to set. ERROR: %v", ip_address, err) + return false + } + + ok, err := set.Test(ip_address) + // output: true + log.Println(ok) + if err != nil { + log.Printf("Failed to added new %v to set. ERROR: %v", ip_address, err) + return false + } + + // ok, _ = set.Test("1.1.1.2") + // output: false + // log.Println(ok) + + // info, _ := set.List() + // output: &{test hash:ip 4 family inet hashsize 1024 maxelem 65536 timeout 3600 216 0 [1.1.1.1 timeout 3599]} + // log.Println(info) + + // _ = set.Del("1.1.1.1") + + // _ = set.Destroy() + return true +} + +// just suit for linux +func processExists(pid string) bool { + if _, err := os.Stat(filepath.Join("/proc", pid)); err == nil { + return true + } + return false +} + +func checkPIDFILEAlreadyExists(path string) error { + if pidByte, err := ioutil.ReadFile(path); err == nil { + pid := strings.TrimSpace(string(pidByte)) + if processExists(pid) { + return fmt.Errorf("ensure the process:%s is not running pid file:%s", pid, path) + } + } + return nil +} + +// NewPIDFile create the pid file +// path specified under production pidfile, file content pid +func NewPIDFile(path string) (*PIDFile, error) { + if err := checkPIDFILEAlreadyExists(path); err != nil { + return nil, err + } + + if err := os.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil { + return nil, err + } + if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil { + return nil, err + } + return &PIDFile{path: path}, nil +} + +func main() { + CheckIPSet() + // Создадим и откроем на запись файл для логов + var alarm_log_file string + if os.Getenv("ALARM_LOG_FILE") == "" { + alarm_log_file = "alarmaction.log" + } else { + alarm_log_file = os.Getenv("ALARM_LOG_FILE") + } + + f, err := os.OpenFile(alarm_log_file, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + + log.SetOutput(f) + + // Создание PID файла + _, err = NewPIDFile(PID_file_path) + if err != nil { + log.Println("error to create the pid file failed:", err.Error()) + ProcessExit(1) + } + + // Проверка переменных окружения + if os.Getenv("PGHOST") == "" { + fmt.Println("Send error: make sure environment variables `PGHOST` was set") + ProcessExit(1) + } + if os.Getenv("PGUSER") == "" { + fmt.Println("Send error: make sure environment variables `PGUSER` was set") + ProcessExit(1) + } + if os.Getenv("PGPASSWORD") == "" { + fmt.Println("Send error: make sure environment variables `PGPASSWORD` was set") + ProcessExit(1) + } + if os.Getenv("PGDATABASE") == "" { + fmt.Println("Send error: make sure environment variables `PGDATABASE` was set") + ProcessExit(1) + } + // if os.Getenv("HOSTNAME") == "" { + // fmt.Println("Send error: make sure environment variables `HOSTNAME` was set") + // ProcessExit(1) + // } + + hostname, err := os.Hostname() + if err != nil { + fmt.Println(err) + ProcessExit(1) + } + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + ProcessExit(1) + } + + sChain := "auto_blocked" + dChain := "DROP" + + // if !IpTablesCreateChain(sChain) { + // log.Println("Iptables chain", sChain, "does not exists") + // ProcessExit(1) + // } + + var alarm Alarm + + alarm.Hostname = strings.Split(hostname, ".")[0] + alarm.Action_timestamp = time.Unix(time.Now().Unix(), 0) + alarm.Alarm_type = 4 + + res := PgSelectAlarmRemoteIP(alarm) + var id int32 + var ip netip.Prefix + var filter_id int32 + for res.Next() { + values, _ := res.Values() + id = values[0].(int32) + ip = values[1].(netip.Prefix) + filter_id = PgSelectFilterIP(ip) + log.Println(filter_id) + if filter_id > 0 { + log.Println("The ip address", ip, "is conained in filter table") + log.Println(PgUpdateAlarm(conn, alarm, id)) + // continue + } else { + fmt.Println(id, ip) + if BlockIP(ip, sChain, dChain) { + log.Println("The ip address", ip, "was blocked") + alarm.Alarm_action = true + log.Println(PgUpdateAlarm(conn, alarm, id)) + } + } + } + ProcessExit(0) +} diff --git a/database.sql.j2 b/database.sql.j2 new file mode 100644 index 0000000..494a49c --- /dev/null +++ b/database.sql.j2 @@ -0,0 +1,148 @@ +-- DROP DATABASE {{ db_name }}; + +-- CREATE DATABASE {{ db_name }} +-- WITH +-- OWNER = {{ db_user }} +-- ENCODING = 'UTF8' +-- LC_COLLATE = 'en_US.utf8' +-- LC_CTYPE = 'en_US.utf8' +-- TABLESPACE = pg_default +-- CONNECTION LIMIT = -1; + +CREATE SEQUENCE public.alarm_id_seq + INCREMENT 1 + START 1 + MINVALUE 1 + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE public.alarm_id_seq + OWNER TO {{ db_user }}; + + +CREATE SEQUENCE public.mail_accounts_id_seq + INCREMENT 1 + START 1 + MINVALUE 1 + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE public.mail_accounts_id_seq + OWNER TO {{ db_user }}; + +CREATE SEQUENCE public.mail_sessions_id_seq + INCREMENT 1 + START 1 + MINVALUE 1 + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE public.mail_sessions_id_seq + OWNER TO {{ db_user }}; + +CREATE SEQUENCE public.sprav_id_seq + INCREMENT 1 + START 1 + MINVALUE 1 + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE public.sprav_id_seq + OWNER TO {{ db_user }}; + + +CREATE SEQUENCE public.filter_id_seq + INCREMENT 1 + START 1 + MINVALUE 1 + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE public.filter_id_seq + OWNER TO mail_alarm; + +CREATE TABLE IF NOT EXISTS public.alarm +( + id integer NOT NULL DEFAULT nextval('alarm_id_seq'::regclass), + mail_account_id integer, + remote_ip inet, + country_code character(6) COLLATE pg_catalog."default", + last_time character varying COLLATE pg_catalog."default", + last_check_timestamp timestamp without time zone, + alarm_type integer, + alarm_action boolean, + action_timestamp timestamp without time zone, + mail_action_timestamp timestamp without time zone, + mail2_action_timestamp timestamp without time zone, + CONSTRAINT alarm_pkey PRIMARY KEY (id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.alarm + OWNER to {{ db_user }}; + + +CREATE TABLE IF NOT EXISTS public.mail_accounts +( + id integer NOT NULL DEFAULT nextval('mail_accounts_id_seq'::regclass), + client_name character varying(100) COLLATE pg_catalog."default" NOT NULL, + client_account character varying(100) COLLATE pg_catalog."default", + remote_ip inet, + country_code character(3) COLLATE pg_catalog."default", + CONSTRAINT mail_accounts_pkey PRIMARY KEY (id, client_name), + CONSTRAINT client_name UNIQUE (client_name) +) + +TABLESPACE pg_default; + +ALTER TABLE public.mail_accounts + OWNER to {{ db_user }}; + +CREATE TABLE IF NOT EXISTS public.mail_sessions +( + id integer NOT NULL DEFAULT nextval('mail_sessions_id_seq'::regclass), + mail_account_id integer, + remote_ip inet, + sessions_count integer, + country_code character(3) COLLATE pg_catalog."default", + last_time character varying COLLATE pg_catalog."default", + last_check_timestamp timestamp without time zone, + mail_reason character varying COLLATE pg_catalog."default", + CONSTRAINT mail_sessions_pkey PRIMARY KEY (id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.mail_sessions + OWNER to {{ db_user }}; + +CREATE TABLE IF NOT EXISTS public.sprav +( + id integer NOT NULL DEFAULT nextval('sprav_id_seq'::regclass), + name character varying COLLATE pg_catalog."default", + sprav_type character(30) COLLATE pg_catalog."default", + CONSTRAINT sprav_pkey PRIMARY KEY (id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.sprav + OWNER to {{ db_user }}; + + +-- DROP TABLE public.filter; + +CREATE TABLE IF NOT EXISTS public.filter +( + id integer NOT NULL DEFAULT nextval('filter_id_seq'::regclass), + remote_ip inet NOT NULL, + description character varying(100) COLLATE pg_catalog."default", + CONSTRAINT filter_pkey PRIMARY KEY (id, remote_ip) +) + +TABLESPACE pg_default; + +ALTER TABLE public.filter + OWNER to mail_alarm; + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d562287 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3' + +services: + zimbra-alarm: + # $IMAGE_PATH и $RELEASE_VERSION определены в .gitlab-ci.yml + image: $IMAGE_PATH/zimbra-alarm:$RELEASE_VERSION + environment: + - ELASTICSEARCH_URL=https://${ELASTIC_USER}:${ELASTIC_PASSWORD}@${ELASTIC_HOST}:9200 + - PGHOST=${PGHOST:-inf-db01.corp.samsonopt.ru} + - PGDATABASE=${PGDATABASE:-mail_alarm} + - PGUSER=${PGUSER:-mail_alarm} + - PGPASSWORD=${PGPASSWORD} + - REQUEST_TIME_RANGE=${REQUEST_TIME_RANGE:-4} + - REQUEST_ROWS=${REQUEST_ROWS:-10000} + - TZ=Europe/Moscow + restart: always + build: + context: . + logging: + # driver: "syslog" + options: + max-size: "10m" + max-file: "5" diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..35a1984 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -u + +while true ;do + /go/bin/zimbra-alarm --indexname "maillog_mailbox" -timerange "${REQUEST_TIME_RANGE}m" -operation es-request-data -pg-insert -request-rows ${REQUEST_ROWS} + sleep 120 +done diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8303ee2 --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module my-elasticsearch-app + +go 1.19 + +require github.com/elastic/go-elasticsearch/v7 v7.17.1 + +require ( + github.com/coreos/go-iptables v0.6.0 // indirect + github.com/gonetx/ipset v0.1.0 // indirect + github.com/hpcloud/tail v1.0.0 // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect +) + +require ( + github.com/adubkov/go-zabbix v0.0.0-20170118040903-3c6a95ec4fdc + github.com/cavaliercoder/go-zabbix v0.0.0-20210304010121-96120c17dd42 // indirect + github.com/elastic/go-elasticsearch/v8 v8.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.12.0 + github.com/jackc/pgx/v4 v4.16.1 // indirect + github.com/jackc/pgx/v5 v5.0.0-beta.3 + github.com/nixys/nxs-go-zabbix/v5 v5.0.0 // indirect + github.com/opensearch-project/opensearch-go/v2 v2.0.0 // indirect + github.com/rday/zabbix v0.0.0-20170517233925-1cf60ccd42f9 // indirect + golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect + golang.org/x/text v0.3.7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..917550a --- /dev/null +++ b/go.sum @@ -0,0 +1,232 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/adubkov/go-zabbix v0.0.0-20170118040903-3c6a95ec4fdc h1:gqqI4ZPa7uwK+gX9Zgk2AweAh+2dX0FpETcXTsA2TrE= +github.com/adubkov/go-zabbix v0.0.0-20170118040903-3c6a95ec4fdc/go.mod h1:ihDXRSVen590YHlXIrv00CcmRrL6pUho/Iwm3ZmM8n8= +github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= +github.com/cavaliercoder/go-zabbix v0.0.0-20210304010121-96120c17dd42 h1:S+MAp8YOH/TkyOTHa1/z/reraTs7fJL0xBOx5+3bA78= +github.com/cavaliercoder/go-zabbix v0.0.0-20210304010121-96120c17dd42/go.mod h1:o9iZ0ep18zjkTdG1yoCmBZSMAWo2qUXVMxqmEl+6GLo= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c h1:onA2RpIyeCPvYAj1LFYiiMTrSpqVINWMfYFRS7lofJs= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= +github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= +github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20210322101442-a3e161131102 h1:o4HAbzLv9EOWc6ue8fyB00AcE78Y3fp8FD9RYy0z73M= +github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20210322101442-a3e161131102/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/go-elasticsearch/v7 v7.17.1 h1:49mHcHx7lpCL8cW1aioEwSEVKQF3s+Igi4Ye/QTWwmk= +github.com/elastic/go-elasticsearch/v7 v7.17.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/go-elasticsearch/v8 v8.3.0 h1:RF4iRbvWkiT6UksZ+OwSLeCEtBg/HO8r88xNiSmhb8U= +github.com/elastic/go-elasticsearch/v8 v8.3.0/go.mod h1:Usvydt+x0dv9a1TzEUaovqbJor8rmOHy5dSmPeMAE2k= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gonetx/ipset v0.1.0 h1:LFkRdTbedg2UYXFN/2mOtgbvdWyo+OERrwVbtrPVuYY= +github.com/gonetx/ipset v0.1.0/go.mod h1:AwNAf1Vtqg0cJ4bha4w1ROX5cO/8T50UYoegxM20AH8= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= +github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= +github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v5 v5.0.0-beta.3 h1:/fvyxKQQVrEgD6elYv2Fa0L16ytVn8Ll18k1XQ/yaGw= +github.com/jackc/pgx/v5 v5.0.0-beta.3/go.mod h1:QJ8xU09HYKHOccHeisi/6sXeRG4dd3AxuV7cmKET4WA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle/v2 v2.0.0-beta.1/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mitchellh/mapstructure v1.3.0 h1:iDwIio/3gk2QtLLEsqU5lInaMzos0hDTz8a6lazSFVw= +github.com/mitchellh/mapstructure v1.3.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nixys/nxs-go-zabbix/v5 v5.0.0 h1:AvVNPs/w2xSampQn25KnvNOx49O9mE5KuIlLrSTXpnE= +github.com/nixys/nxs-go-zabbix/v5 v5.0.0/go.mod h1:9R4cq2L8wHvVAz0ZfNIvzyXRUGCsrJOd7HvWiRoz6TI= +github.com/opensearch-project/opensearch-go/v2 v2.0.0 h1:Ij3CpuHwey29cYPVMgi5h1pWBH2O0JaTXsa4c7pqhK4= +github.com/opensearch-project/opensearch-go/v2 v2.0.0/go.mod h1:G3kbnV+SeVf4QTbNcrT7Ga3FCsavtp5NQfdRelJikIQ= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rday/zabbix v0.0.0-20170517233925-1cf60ccd42f9 h1:ARUuqpY5LjSR4//P6TolxJP81zjO+qNuOlEVD4mvIfs= +github.com/rday/zabbix v0.0.0-20170517233925-1cf60ccd42f9/go.mod h1:IIFS/h+mpikvp6WVUQwg9fO8IlGy7q3LE1wt3MeWhvg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/zimbra-alarm.go b/zimbra-alarm.go new file mode 100644 index 0000000..d4ad881 --- /dev/null +++ b/zimbra-alarm.go @@ -0,0 +1,1107 @@ +//------------------------------------------------------------------------------ +// Getting and monitoring Elasticsearch indices and send the metrics into zabbix +//------------------------------------------------------------------------------- +// Author: Sergey Kalinin +//------------------------------------------------------------------------------- +// ZABBIX_SERVER="zabbix" +// ZABBIX_PORT="10051" +// ZABBIX_HOST="elastic cluster" +// ELASTICSEARCH_URL="https://user:pass@elastic:9200" +//------------------------------------------------------------------------------- + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "net/netip" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" + "reflect" + + "github.com/adubkov/go-zabbix" + // "github.com/elastic/go-elasticsearch/v7" + // "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgtype" +) + +type json_records struct { + INDEX string + RECORD_COUNT int +} + +type Mail_records struct { + id int32 + client_name string + client_account string + //remote_ip pgtype.Inet + remote_ip string + country_code string + check_time time.Time + session_time string + mail_reason string +} + +type ESIndex struct { + es_index_name string + es_request_time_range string + es_request_rows_quantity int +} + +// type ESQuery struct { + // Query struct { + // Bool struct { + // Filter []struct { + // Range struct { + // Timestamp struct { + // Gt string `json:"gt"` + // } `json:"@timestamp"` + // } `json:"range"` + // Match struct { + // MailCommand string `json:"mail.command"` + // } `json:"match"` + // } `json:"filter"` + // } `json:"bool"` + // } `json:"query"` + // Size int `json:"size"` +// } +type ESQuery struct { + Query struct { + Bool struct { + Filter []map[string]interface{} `json:"filter"` + } `json:"bool"` + } `json:"query"` + Size int `json:"size"` +} + +type MailSession struct { + id int32 + mail_account_id int32 + sessions_count int32 + country_code string + remote_ip netip.Prefix + last_time string + check_time time.Time + alarm_type int +} + +type Config struct { + sessions_count int32 + country_code string +} + +var Exclude_index_list string +var conn *pgx.Conn + +// Определяем входит ли строка в slice +func StringContains(a []string, x string) bool { + for _, n := range a { + if x == n { + return true + } + } + return false +} + +func isNil(i interface{}) bool { + if i == nil { + return true + } + switch reflect.TypeOf(i).Kind() { + case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: + return reflect.ValueOf(i).IsNil() + } + return false +} +// Проверка вхождения имени индекса (префикса) в список исключений +func IndexPrefixExclude(index_name string) bool { + exclude_index_name := strings.Split(Exclude_index_list, ",") + // log.Println(Exclude_index_list) + for _, n := range exclude_index_name { + if strings.HasPrefix(index_name, n) { + return true + } + } + return false +} + +func EsSearch(index_name string, query ESQuery) map[string]interface{} { + var ( + r map[string]interface{} + ) + cfg := opensearch.Config{ + // Addresses: []string{ + // "http://localhost:9200", + // "http://localhost:9201", + // }, + // Username: "foo", + // Password: "bar", + Transport: &http.Transport{ + MaxIdleConnsPerHost: 10, + ResponseHeaderTimeout: 60 * time.Second, + DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, + TLSClientConfig: &tls.Config{ + MaxVersion: tls.VersionTLS13, + InsecureSkipVerify: true, + }, + }, + } + + es, err := opensearch.NewClient(cfg) + if err != nil { + log.Fatalf("Error creating the client: %s", err) + } + res, err := es.Info() + if err != nil { + log.Fatalf("Error getting response: %s", err) + } + defer res.Body.Close() + // Check response status + if res.IsError() { + log.Fatalf("Error: %s", res.String()) + } + var buf bytes.Buffer + // query := map[string]interface{}{ + // "query": map[string]interface{}{ + // "range": map[string]interface{}{ + // "@timestamp": map[string]interface{}{ + // // "gt": "now-1h", + // "gt": request_time_range, + // }, + // } + // "match": map[string]interface{}{ + // "mail.command": "Auth", + // }, + // }, + // "size": es_request_rows_quantity, + // "sort": map[string]interface{}{ + // "@timestamp": map[string]interface{}{ + // "order": "desc", + // }, + // }, + // } + + a, _ := json.Marshal(query) + fmt.Println(string(a)) + + if err := json.NewEncoder(&buf).Encode(query); err != nil { + log.Fatalf("Error encoding query: %s", err) + } + + // Perform the search request. + res, err = es.Search( + es.Search.WithContext(context.Background()), + es.Search.WithIndex(index_name+"*"), + es.Search.WithBody(&buf), + es.Search.WithTrackTotalHits(true), + es.Search.WithPretty(), + ) + if err != nil { + log.Fatalf("Error getting response: %s", err) + } + defer res.Body.Close() + + if res.IsError() { + var e map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&e); err != nil { + log.Fatalf("Error parsing the response body: %s", err) + } else { + // Print the response status and error information. + log.Fatalf("[%s] %s: %s", + res.Status(), + e["error"].(map[string]interface{})["type"], + e["error"].(map[string]interface{})["reason"], + ) + } + } + + if err := json.NewDecoder(res.Body).Decode(&r); err != nil { + log.Printf("Error parsing the response body:", err) + } + + // json_data, _ := json.Marshal(r) + // fmt.Println(string(json_data)) + + return r +} + +func EsIndexDiscover(es_index_list []string) string { + result_discovery_json := string("{\"data\": [") + + for _, value := range es_index_list { + // result_discovery_json = result_discovery_json + "{\"{#INDEX}\":\"" + value + "\",\"{#ITEM_TYPE}\":\"Records count\"" + "}," + result_discovery_json = result_discovery_json + "{\"{#INDEX}\":\"" + value + "\"" + "}," + } + result_discovery_json = strings.TrimRight(result_discovery_json, ",") + "]}" + // fmt.Println(result_discovery_json) + return (result_discovery_json) +} + +// func EsIndexRecordsCount(es_index_list []string, es_request_time_range string, es_request_rows_quantity int) { + // // fmt.Println(es_request_time_range) + // result_index := make(map[int]json_records) + // j := int(0) + // for _, value := range es_index_list { + // r := EsSearch(value, es_request_time_range, es_request_rows_quantity) + // result := int(r["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) +// + // result_index[j] = json_records{value, result} + // j++ + // } + // log.Println(result_index) +// } +// +func EsClusterInfo(es *opensearch.Client) { + var ( + r map[string]interface{} + ) + // 1. Get cluster info + res, err := es.Info() + if err != nil { + log.Fatalf("Error getting response: %s", err) + } + defer res.Body.Close() + // Check response status + if res.IsError() { + log.Fatalf("Error: %s", res.String()) + } + // Deserialize the response into a map. + if err := json.NewDecoder(res.Body).Decode(&r); err != nil { + log.Fatalf("Error parsing the response body: %s", err) + } + // Print client and server version numbers. + fmt.Printf("Client: %s\n", opensearch.Version) + fmt.Printf("Client: %s\n", opensearch.Version) + fmt.Printf("Server: %s\n", r["version"].(map[string]interface{})["number"]) +} + +func EsIndexFieldExists(es *opensearch.Client, index_name string, field_name string) bool { + // Check if the filed exists into index + var ( + fields_request map[string]interface{} + fields []string + indices []string + field_exists bool + ) + fields = append(fields, field_name) + indices = append(indices, index_name) + + req := opensearchapi.FieldCapsRequest{ + // Index: []string{}, + Index: indices, + Fields: fields, + } + + res, err := req.Do(context.Background(), es) + if err != nil { + log.Fatalf("ERROR: %s", err) + } + if err := json.NewDecoder(res.Body).Decode(&fields_request); err != nil { + log.Fatalf("->Error parsing the response body: %s", err) + } + + for i, f := range fields_request { + if i == "fields" { + // if f.(map[string]interface{})[`@timestamp`] == nil { + if f.(map[string]interface{})[field_name] == nil { + field_exists = false + } else { + field_exists = true + } + } + } + return (field_exists) +} + +func EsGetIndices(es *opensearch.Client) []string { + //Get indices list + // log.Println("sjsjsjsj") + var ( + index_request []map[string]interface{} + es_index_list []string + ) + req := opensearchapi.CatIndicesRequest{ + // Index: []string{}, + Format: "JSON", + // ExpandWildcards, + } + + res, err := req.Do(context.Background(), es) + if err != nil { + log.Fatalf("ERROR: %s", err) + } + // log.Println(res) + + if err := json.NewDecoder(res.Body).Decode(&index_request); err != nil { + log.Fatalf("Error parsing the response body: %s", err) + } + for _, hit := range index_request { + m := regexp.MustCompile("-[0-9]") + // позиция вхождения регулярки в имени индекса + i := m.FindStringIndex(hit["index"].(string)) + // наименование индекса + index_name := hit["index"].(string) + // log.Println(index_name) + // Checking the @timestamp field exists + // if !EsIndexFieldExists(es, index_name, `@timestamp`) { + // continue + // // fmt.Println(EsIndexFieldExists(es, index_name)) + // } + + // if strings.HasPrefix(index_name, "apdex") { + // continue + // // fmt.Println(EsIndexFieldExists(es, index_name)) + // } + // Итоговое наименование индекса + var index_name_short string + // Исключаем индексы от кибаны (.kibana-*) и обрезаем цифры (даты и версии) + if !IndexPrefixExclude(index_name) { + if i != nil { + index_name_short = strings.TrimSpace(index_name[0:i[0]]) + } else { + index_name_short = strings.TrimSpace(index_name) + } + if !StringContains(es_index_list, index_name_short) { + es_index_list = append(es_index_list, index_name_short) + } + } else { + log.Println("Index", index_name, "has name from exclude lists") + } + } + log.Println(es_index_list) + + return (es_index_list) +} + +func ZabbixSender(zabbix_host string, metric_name string, metric_value string) bool { + var ( + metrics []*zabbix.Metric + zabbix_port int + err error + ) + zabbix_server := os.Getenv("ZABBIX_SERVER") + + // Read and checkin zabbix server and port + if zabbix_server == "" { + log.Println("Login error: make sure environment variables `ZABBIX_SERVER` are defined") + os.Exit(1) + } + if os.Getenv("ZABBIX_PORT") == "" { + log.Println("Login error: make sure environment variables `ZABBIX_PORT` are defined") + os.Exit(1) + } + if zabbix_port, err = strconv.Atoi(os.Getenv("ZABBIX_PORT")); err != nil { + log.Println(zabbix_port, "Zabbix port value error") + } + + metrics = append(metrics, zabbix.NewMetric(zabbix_host, metric_name, metric_value, time.Now().Unix())) + metrics = append(metrics, zabbix.NewMetric(zabbix_host, "status", "OK")) + + // Create instance of Packet class + packet := zabbix.NewPacket(metrics) + // fmt.Println(zabbix_host, metric_name, metric_value, metrics) + // Send packet to zabbix + z := zabbix.NewSender(zabbix_server, zabbix_port) + res, err := z.Send(packet) + log.Println(string(res)) + return true +} + +func PgSearch(conn *pgx.Conn, mail_data Mail_records) int32 { + var id int32 + var client_name pgtype.Varchar + // var remote_ip pgtype.CIDR + // var country_code pgtype.Varchar + + // fmt.Println("Select data from DB:", mail_data.client_name) + + err := conn.QueryRow(context.Background(), "select id, client_name from mail_accounts where client_name = $1", mail_data.client_name).Scan(&id, &client_name) + //err = conn.QueryRow(context.Background(), "select id, client_name, remote_ip, country_code from mail_accounts where") + if err != nil { + fmt.Fprintf(os.Stderr, "Query select account failed: %v\n", err) + // id = PgInsertAccount(conn, mail_data) + log.Println("Select", mail_data.client_name, "error:", err) + if err.Error() == "no rows in result set" { + id = 0 + } else { + os.Exit(1) + } + } + // fmt.Println(">>>", id, client_name.String) + return id +} + +func PgInsertAccount(conn *pgx.Conn, mail_data Mail_records) int32 { + var id int32 + // var client_name pgtype.Varchar + // var remote_ip pgtype.CIDR + // var country_code pgtype.Varchar + // fmt.Println("Insert data into DB") + + err := conn.QueryRow(context.Background(), "insert into mail_accounts(client_name) values($1) returning id", mail_data.client_name).Scan(&id) + if err != nil { + fmt.Fprintf(os.Stderr, "Query insert account failed: %v\n", err) + } + //fmt.Println(id) + return id +} + +func PgSearchMailSession(conn *pgx.Conn, mail_data Mail_records) (int32, int32) { + var id int32 + var session_count int32 + // var client_name pgtype.Varchar + // var remote_ip pgtype.CIDR + var country_code pgtype.Varchar + + // log.Println("Select data from DB:", "select id, sessions_count, country_code from mail_sessions where mail_account_id =", mail_data.id, "and remote_ip =",mail_data.remote_ip, "and last_time =", mail_data.session_time, "and last_check_timestamp =", mail_data.check_time) + + err := conn.QueryRow(context.Background(), "select id, sessions_count, country_code from mail_sessions where mail_account_id = $1 and remote_ip = $2 and last_check_timestamp = $3", mail_data.id, mail_data.remote_ip, mail_data.check_time).Scan(&id, &session_count, &country_code) + //err = conn.QueryRow(context.Background(), "select id, client_name, remote_ip, country_code from mail_accounts where")\ + + // log.Println("Selected session data: ", id, session_count, country_code) + if err != nil { + log.Println("Query session PgSearchMailSession failed:", err) + // id = PgInsertAccount(conn, mail_data) + // log.Println("Select", mail_data.client_name, "error:", err) + if err.Error() == "no rows in result set" { + id = 0 + } else { + os.Exit(1) + } + } + // fmt.Println(">>>", id, client_name.String) + return id, session_count +} + +// Alarm_type +// 1 "Неразрешенный код страны" "alarm" +// 2 "Количество сессий больше заданного" "alarm" +// 3 "Вход в аккаунт более чем с одного IP" "alarm" +// 4 "Вход более чем в один аккаунт" "alarm" + +func PgSearchMailSessionAnomaly(conn *pgx.Conn, last_check_timestamp time.Time, conf Config) []MailSession { + // log.Println("Select data from DB:", "select id, mail_account_id, sessions_count, country_code, remote_ip, last_time from mail_sessions where last_check_timestamp =", last_check_timestamp) + + rows, err_select := conn.Query(context.Background(), "select id, mail_account_id, sessions_count, country_code, remote_ip, last_time from mail_sessions where last_check_timestamp = $1", last_check_timestamp) + if err_select != nil { + log.Println("Query sessions failed:", err_select) + + os.Exit(1) + } + var res []MailSession + for rows.Next() { + var s MailSession + values, _ := rows.Values() + s.id = values[0].(int32) + s.mail_account_id = values[1].(int32) + s.sessions_count = values[2].(int32) + s.country_code = strings.TrimSpace(values[3].(string)) + s.remote_ip = values[4].(netip.Prefix) + s.last_time = values[5].(string) + s.check_time = last_check_timestamp + + log.Println(s.id, s.mail_account_id, s.sessions_count, s.country_code, s.remote_ip, s.last_time) + if s.country_code != conf.country_code && s.country_code != "" { + s.alarm_type = 1 + log.Println("ALARM!!!", s.mail_account_id, s.sessions_count, s.country_code, s.remote_ip) + res = append(res, s) + } + if s.sessions_count >= conf.sessions_count { + // s.alarm_type = "Количество соединений больше " + string(conf.sessions_count) + s.alarm_type = 2 + log.Println("ALARM!!!", s.mail_account_id, s.sessions_count, s.country_code, s.remote_ip) + res = append(res, s) + // alarm_id := PgInsertMailAlarm(conn, s, "Код страны GeoIP не разрешен") + // log.Println("Alarm insert with id:", alarm_id) + } + } + + return res +} + +func PgSelectAlarm(conn *pgx.Conn, mail_session MailSession) (int32) { + var id int32 + + // log.Println("Select data from DB:", "select id, sessions_count, country_code from mail_sessions where mail_account_id =", mail_data.id, "and remote_ip =",mail_data.remote_ip, "and last_time =", mail_data.session_time, "and last_check_timestamp =", mail_data.check_time) + + err := conn.QueryRow(context.Background(), "select id from alarm where mail_account_id = $1 and remote_ip = $2 and alarm_type = $3", mail_session.mail_account_id, mail_session.remote_ip, mail_session.alarm_type).Scan(&id) + log.Println("Selected alarm data: ", mail_session.mail_account_id, mail_session.remote_ip) + if err != nil { + log.Println("Select alarm with account_id", mail_session.mail_account_id, " and remote ip", mail_session.remote_ip, "error:", err) + if err.Error() == "no rows in result set" { + id = 0 + } else { + os.Exit(1) + } + } + return id +} +func PgSelectAlarmAccount(conn *pgx.Conn, mail_session MailSession) int32 { + var s int32 + var i int32 + + // log.Println("Select data from DB:", "select id, sessions_count, country_code from mail_sessions where mail_account_id =", mail_data.id, "and remote_ip =",mail_data.remote_ip, "and last_time =", mail_data.session_time, "and last_check_timestamp =", mail_data.check_time) + query := fmt.Sprintf("select id from alarm where mail_account_id = %v and alarm_type = %v", mail_session.mail_account_id, mail_session.alarm_type) + log.Println("PgSelectAlarmAccount execute query:", query) + rows, err_select := conn.Query(context.Background(), query) + if err_select != nil { + log.Println("Query alarms PgSelectAlarmAccount failed:", err_select) + os.Exit(1) + } + + if len(rows.FieldDescriptions()) != 0 { + i = 0 + var id int32 + for rows.Next() { + values, _ := rows.Values() + id = values[0].(int32) + log.Println("\t found records with id:", id) + i++ + } + // если выбрана только одна запись возвращаем ее id, + // если несколько - то возвращаем количество записей + if i == 1 { + s = id + } else { + s = i + } + + } else { + s = 0 + } + + return s +} + +// Выборка записей по конкретному IP +// запросы разные в зависимости от типа тревоги +func PgSelectAlarmRemoteIP(conn *pgx.Conn, mail_session MailSession) (int32) { + var s int32 + var query string + + // В зависимости от типа тревоги функция возвращает либо количество записей либо ID записи + switch mail_session.alarm_type { + case 1: + // По уму тут можно упростить добавив в запрос count() хотя не факт что прокатит + query = fmt.Sprintf("select mail_account_id, remote_ip, alarm_type from alarm where remote_ip = '%v' and alarm_type = %v group by mail_account_id, remote_ip, alarm_type", mail_session.remote_ip, mail_session.alarm_type) + log.Println("PgSelectAlarmRemoteIP execute query:",query) + rows, err_select := conn.Query(context.Background(), query) + + if err_select != nil { + log.Println("Query alarms PgSelectAlarmRemoteIP failed:", err_select) + os.Exit(1) + } + if len(rows.FieldDescriptions()) != 0 { + var i, id int32 + i = 0 + for rows.Next() { + values, _ := rows.Values() + id = values[0].(int32) + log.Println("\t found records with id:", id) + i++ + } + s = i + + } else { + s = 0 + } + case 4: + query = fmt.Sprintf("select id from alarm where remote_ip = '%v' and alarm_type = %v", mail_session.remote_ip, mail_session.alarm_type) + log.Println(query) + var id int32 + err := conn.QueryRow(context.Background(), query).Scan(&id) + + if err != nil { + log.Println("Query alarms PgSelectAlarmRemoteIP failed:", mail_session.remote_ip, "error:", err) + if err.Error() == "no rows in result set" { + id = 0 + } else { + os.Exit(1) + } + } else { + s = id + } + } + return s +} + + +func PgInsertAlarm(conn *pgx.Conn, mail_data MailSession, conf Config) int32 { + var id int32 + + log.Println("Insert data", mail_data, "into alarm DB") + err := conn.QueryRow(context.Background(), "insert into alarm(mail_account_id, remote_ip, country_code, last_time, last_check_timestamp, alarm_type, alarm_action) values($1, $2, $3, $4, $5, $6, false) returning id", mail_data.mail_account_id, mail_data.remote_ip, mail_data.country_code, mail_data.last_time, mail_data.check_time, mail_data.alarm_type).Scan(&id) + if err != nil { + log.Printf("Insert alarm data failed:", err) + } + return id +} + +func PgInsertAlarmRemoteIP(conn *pgx.Conn, mail_data MailSession, conf Config) int32 { + var id int32 + + log.Println("Insert data", mail_data, "into alarm DB") + err := conn.QueryRow(context.Background(), "insert into alarm(remote_ip, country_code, last_time, last_check_timestamp, alarm_type, alarm_action) values($1, $2, $3, $4, $5, false) returning id", mail_data.remote_ip, mail_data.country_code, mail_data.last_time, mail_data.check_time, mail_data.alarm_type).Scan(&id) + if err != nil { + log.Printf("Insert alarm data failed:", err) + } + return id +} + +func PgInsertAlarmAccount(conn *pgx.Conn, mail_data MailSession, conf Config) int32 { + var id int32 + + log.Println("Insert data", mail_data, "into alarm DB") + err := conn.QueryRow(context.Background(), "insert into alarm(mail_account_id, last_time, last_check_timestamp, alarm_type, alarm_action) values($1, $2, $3, $4, false) returning id", mail_data.mail_account_id, mail_data.last_time, mail_data.check_time, mail_data.alarm_type).Scan(&id) + if err != nil { + log.Printf("Insert alarm data failed:", err) + } + return id +} + +func PgUpdateAlarm(conn *pgx.Conn, mail_session MailSession, conf Config, id int32) int32 { + log.Println("Update alarm data", mail_session) + err := conn.QueryRow(context.Background(), "update alarm set last_time=$1, last_check_timestamp=$2 where id=$3 returning id", mail_session.last_time, mail_session.check_time, id).Scan(&id) + if err != nil { + log.Printf("Update alarm data failed: %v\n", err) + } + + return id +} + + +func PgInsertMailSession(conn *pgx.Conn, mail_data Mail_records) int32 { + var id int32 + + sessions_count := 1 + + // log.Println("Insert data", mail_data, "into session DB") + err := conn.QueryRow(context.Background(), "insert into mail_sessions(mail_account_id, remote_ip, sessions_count, last_time, country_code, last_check_timestamp, mail_reason) values($1, $2, $3, $4, $5, $6, $7) returning id", mail_data.id, mail_data.remote_ip, sessions_count, mail_data.session_time, mail_data.country_code, mail_data.check_time, mail_data.mail_reason).Scan(&id) + if err != nil { + log.Printf("Insert session data failed:", err) + } + return id +} + +func PgUpdateMailSession(conn *pgx.Conn, mail_data Mail_records, session_id int32, sessions_count int32) int32 { + var id int32 + + sessions_count++ + + // log.Println("Update data", mail_data, "into session DB") + err := conn.QueryRow(context.Background(), "update mail_sessions set sessions_count=$1, last_time=$2, last_check_timestamp=$4, mail_reason=$5 where id=$3 returning id", sessions_count, mail_data.session_time, session_id, mail_data.check_time, mail_data.mail_reason).Scan(&id) + if err != nil { + log.Printf("Update session data failed: %v\n", err) + } + + return id +} + +func doit(mail Mail_records, es_index ESIndex, query ESQuery, c chan int) int { + // Запрос данных из соответствующего индекса и выборка полей mail.{} и geoip.{} + fmt.Println(es_index) + fmt.Println(es_index.es_index_name) + r := EsSearch(es_index.es_index_name, query) + result := r["hits"].(map[string]interface{})["hits"].([]interface{}) + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + // fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + os.Exit(1) + } + for i := range result { + mail_res := result[i].(map[string]interface{})["_source"].(map[string]interface{})["mail"] + geoip :=result[i].(map[string]interface{})["_source"].(map[string]interface{})["geoip"] + if !isNil(mail_res) { + //fmt.Println(">>>>>>", mail.(map[string]interface{})) + // fmt.Println(">>>>>>", mail_res.(map[string]interface{})["client_account"]) + if !isNil(mail_res.(map[string]interface{})["client_account"]) { + mail.client_account = mail_res.(map[string]interface{})["client_account"].(string) + } + + // fmt.Println("", mail_res.(map[string]interface{})["client_name"]) + if !isNil(mail_res.(map[string]interface{})["client_name"]) { + mail.client_name = mail_res.(map[string]interface{})["client_name"].(string) + } + + // fmt.Println("", mail_res.(map[string]interface{})["remote_ip"]) + if !isNil( mail_res.(map[string]interface{})["remote_ip"]) { + mail.remote_ip = mail_res.(map[string]interface{})["remote_ip"].(string) + } + if !isNil( mail_res.(map[string]interface{})["log_datetime"]) { + datetime := mail_res.(map[string]interface{})["log_datetime"].(string) + // tmpl := "2022-08-18 16:17:24,446" + mail.session_time = datetime + // log.Println(">>>>>>>>>>>>>>>>", mail.session_time, datetime) + // 2022-08-18 16:17:24,446 + } + if !isNil(mail_res.(map[string]interface{})["reason"]) { + mail.mail_reason = mail_res.(map[string]interface{})["reason"].(string) + } } + if !isNil(geoip) { + // fmt.Println("", geoip.(map[string]interface{})["country_iso_code"].(string)) + mail.country_code = geoip.(map[string]interface{})["country_iso_code"].(string) + } + if mail.client_name == "" && mail.client_account != "" { + mail.client_name = mail.client_account + } + if (mail.client_name != "") && mail.remote_ip != "" { + log.Println(mail.client_name, mail.remote_ip, mail.country_code, mail.mail_reason) + // fmt.Println(mail.client_name, mail.client_account, mail.remote_ip, mail.country_code) + res := PgSearch(conn, mail) + if res == 0 { + mail.id = PgInsertAccount(conn, mail) + } else { + mail.id = res + } + // log.Println("Insert account", mail.client_name, mail.client_account," sucess with id", mail.id) + + id, session_count := PgSearchMailSession(conn, mail) + if id == 0 { + PgInsertMailSession(conn, mail) + } else { + PgUpdateMailSession(conn, mail, id, session_count) + } + } + + mail.client_account = "" + mail.client_name = "" + mail.remote_ip = "" + mail.country_code = "" + mail_res = nil + } + defer conn.Close(context.Background()) + // Вывод статуса, количество выбранных записей и времени запроса + hits := int(r["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) + log.Println("ES index name:", es_index.es_index_name, "; Query time period:", es_index.es_request_time_range, "; Records count:", hits) + c <- hits + return hits +} + +//--------------------------------------------------------------------------- +func main() { + var ( + operation string + es_index_name string + es_request_time_range string + es_request_rows_quantity int + zabbix_send bool + pg_insert bool + // zabbix_port int + ) + + flag.StringVar(&es_index_name, "indexname", "", "Elasticsearch index name patterns by comma separated, (like \"filebeat\")") + flag.StringVar(&es_request_time_range, "timerange", "1h", "Elasticsearch time range for records count (like 1m, 10h, e.t.c.)") + flag.IntVar(&es_request_rows_quantity, "request-rows", 1, "How many rows will return") + flag.BoolVar(&zabbix_send, "zabbix-send", false, "Send metrics or discovery data into zabbix") + flag.BoolVar(&pg_insert, "pg-insert", false, "Write data into postgres dsatabase") + flag.StringVar(&Exclude_index_list, "exclude-index", ".", "Elasticsearch index name comma-separated list for exluded, (like \"filebeat,mailindex,e.t.c\")") + + flag.StringVar(&operation, "operation", "es-cluster-info", "Opertation type, must be:\n\tes-cluster-info - ES Cluster information (version e.t.c)\n\tes-get-indices - geting all index\n\tes-indices-discover - getting es index pattern list\n\tes-records-count - getting the number of records for a time range for all index pattern\n\tes-index-records-count - getting records count for one index (used with -indexname option\n\tes-request-data - select data from index") + + flag.Parse() + fmt.Println(zabbix_send) + zabbix_host := os.Getenv("ZABBIX_HOST") + if zabbix_host == "" && zabbix_send == true { + fmt.Println("Send error: make sure environment variables `ZABBIX_HOST` was set") + os.Exit(1) + } + + if os.Getenv("ELASTICSEARCH_URL") == "" { + fmt.Println("Send error: make sure environment variables `ELASTICSEARCH_URL` was set") + os.Exit(1) + } + // if es_request_time_range > 24 { + // log.Println("Error: Time range must be a number between 1 and 24") + // os.Exit(1) + // } + if pg_insert { + if os.Getenv("PGHOST") == "" { + fmt.Println("Send error: make sure environment variables `PGHOST` was set") + os.Exit(1) + } + if os.Getenv("PGUSER") == "" { + fmt.Println("Send error: make sure environment variables `PGUSER` was set") + os.Exit(1) + } + if os.Getenv("PGPASSWORD") == "" { + fmt.Println("Send error: make sure environment variables `PGPASSWORD` was set") + os.Exit(1) + } + if os.Getenv("PGDATABASE") == "" { + fmt.Println("Send error: make sure environment variables `PGDATABASE` was set") + os.Exit(1) + } + } + + cfg := opensearch.Config{ + Transport: &http.Transport{ + MaxIdleConnsPerHost: 10, + ResponseHeaderTimeout: 60 * time.Second, + DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, + TLSClientConfig: &tls.Config{ + MaxVersion: tls.VersionTLS13, + InsecureSkipVerify: true, + }, + }, + } + + es, err := opensearch.NewClient(cfg) + if err != nil { + log.Fatalf("Error creating the client: %s", err) + } + // fmt.Println(operation) + switch operation { + case "es-cluster-info": + EsClusterInfo(es) + case "es-get-indices": + // fmt.Println(operation) + for _, index := range EsGetIndices(es) { + fmt.Println(index) + } + case "es-indices-discover": + result_discovery_json := EsIndexDiscover(EsGetIndices(es)) + // fmt.Println(zabbix_send) + if zabbix_send == true { + ZabbixSender(zabbix_host, "indices", result_discovery_json) + } + fmt.Println(result_discovery_json) + case "es-records-count": + // records count for all indices + // EsIndexRecordsCount(EsGetIndices(es), es_request_time_range) + // result_index := make(map[int]json_records) + // j := int(0) + // for _, es_index_name := range EsGetIndices(es) { + // r := EsSearch(es_index_name, es_request_time_range, es_request_rows_quantity) + // result := int(r["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) + // result_index[j] = json_records{es_index_name, result} + // log.Println("ES index name:", es_index_name, "; Query time period:", es_request_time_range, "; Records count:", result) + // if zabbix_send == true { + // ZabbixSender(zabbix_host, `indices.records_count[`+es_index_name+`]`, strconv.Itoa(result)) + // ZabbixSender(zabbix_host, `indices.records_count_time_range[`+es_index_name+`]`, es_request_time_range) + // } + // j++ + // } + case "es-index-records-count": + // records count for one index + // r := EsSearch(es_index_name, es_request_time_range, es_request_rows_quantity) + // result := int(r["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) + // if zabbix_send == true { + // ZabbixSender(zabbix_host, `indices.records_count[`+es_index_name+`]`, strconv.Itoa(result)) + // ZabbixSender(zabbix_host, `indices.records_count_time_range[`+es_index_name+`]`, es_request_time_range) + // } + // log.Println("ES index name:", es_index_name, "; Query time period:", es_request_time_range, "; Records count:", result) + case "es-request-data": + var conf Config + conf.sessions_count = 50 + conf.country_code = "RU" + + + var mail Mail_records + mail.check_time = time.Unix(time.Now().Unix(), 0) + last_check_timestamp := mail.check_time + + // канал для получания информации от процессов (routine) + c := make(chan int) + + var query_auth ESQuery + var query_invalid ESQuery + request_time_range := "now-" + es_request_time_range + + s := `{ + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "gt": "now-5m" + } + } + }, + { + "match": { + "mail.command": "Authentication" + } + }, + { + "match_phrase": { + "log.file.path": "/opt/zimbra/log/mailbox.log" + } + } + ] + } + }, + "size": 1000 + }` + + err = json.Unmarshal([]byte(s), &query_auth) + if err != nil { + fmt.Println("JSON query error:",err) + } + err = json.Unmarshal([]byte(s), &query_invalid) + if err != nil { + fmt.Println("JSON query error:",err) + } + + query_auth.Query.Bool.Filter[0] = map[string]interface{}{ + "range": map[string]interface{}{ + "@timestamp": map[string]string{ + "gt": request_time_range, + }, + }, + } + query_auth.Query.Bool.Filter[1] = map[string]interface{}{ + "match": map[string]string{ + "mail.command": "authentication", + }, + } + + query_auth.Query.Bool.Filter[2] = map[string]interface{}{ + "match_phrase": map[string]string{ + "log.file.path": "/opt/zimbra/log/mailbox.log", + }, + } + // + + query_auth.Size = es_request_rows_quantity + + query_invalid.Query.Bool.Filter[0] = map[string]interface{}{ + "range": map[string]interface{}{ + "@timestamp": map[string]string{ + "gt": request_time_range, + }, + }, + } + query_invalid.Query.Bool.Filter[2] = map[string]interface{}{ + "match_phrase": map[string]string{ + "log.file.path": "/opt/zimbra/log/mailbox.log", + }, + } + // + + query_invalid.Size = es_request_rows_quantity + query_invalid.Query.Bool.Filter[1] = map[string]interface{}{ + "match": map[string]string{ + "mail.reason" : "*password", + }, + } + // Получаем список индексов и делаем список индексов согласно списка шаблонов имени + // заданного с коммандной строки es_index_name_prefix + es_indicies_all := EsGetIndices(es) + + var es_indicies_filtered_list []string + + for _, finded_index := range es_indicies_all { + // fmt.Println(finded_index) + for _, pattern := range strings.Split(es_index_name, ",") { + if strings.HasPrefix(finded_index, pattern) { + es_indicies_filtered_list = append(es_indicies_filtered_list, finded_index) + fmt.Println(">>",finded_index) + } + } + // maillog_mailbox4_filebeat-2023.10.16 + // es_indicies_filtered = strings.TrimPrefix(s, "¡¡¡Hello, ") + } + // запускаем параллельно работы по каждому индексу + for _, index := range es_indicies_filtered_list { + // fmt.Println(index) + var es_index ESIndex + es_index.es_index_name = index + es_index.es_request_time_range = es_request_time_range + es_index.es_request_rows_quantity = es_request_rows_quantity + go doit(mail, es_index, query_auth, c) + go doit(mail, es_index, query_invalid, c) + // doit(mail, es_index, query_auth, c) + // doit(mail, es_index, query_invalid, c) + } + // Читаем записи из канала для каждого индекса и обрабатываем далее + // for _, index := range strings.Split(es_index_name, ",") { + for _, index := range es_indicies_filtered_list { + hits := <-c // Получает значение от канала + log.Println("request for", index, "has finished with", hits, "records") + } + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + fmt.Println(*conn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + os.Exit(1) + } + // Выбираем аномалии согласно конфига + result := PgSearchMailSessionAnomaly(conn, last_check_timestamp, conf) + + // --- Типы тревог ---- + // 1 "Неразрешенный код страны" + // 2 "Количество сессий больше заданного" + // 3 "Вход в аккаунт более чем с одного IP" + // 4 "Вход более чем в один аккаунт" + + // Задачи: + // 1. выбрать текущие сессии с geoip не равным RU (mail_session, alarm_type = 1) + // 2. выбрать текущие сессии с количеством больше заданного (mail_sessions, alarm_type = 2) + // 3. выбрать тревоги за все периоды по каждому ip из текущих сессий + // - входы с одного адреса в несколько аккаунтов (alarm, alarm_type = 4) + // 4. Выбрать тревоги за все периоды по каждому аккаунту из текущих сессий + // - входы в один аккаунт с нескольких адресов (alarm, alarm_type = 3) + // Если тревога для данного адреса или аккаунта заданного типа уже есть + // то просто обновить запись с новой меткой времени. + // При выборке тревог выставить alarm_action = false + + if len(result) != 0 { + log.Println("Anomaly finded", result) + for i := range result { + // Смотрим есть ли уже запись для данной аномалии + // если есть то обновляем если нет - добавляем + id := PgSelectAlarm(conn, result[i]) + if id == 0 { + log.Println("Alarm insert with id", PgInsertAlarm(conn, result[i], conf)) + } else { + log.Println("Alarm type",result[i].alarm_type,"with id", PgUpdateAlarm(conn, result[i], conf, id), "was updated") + } + // Выбираем список тревог для определенного аккаунта и типа тревоги = 1. + // Если возвращается больше 1-й записи то генерим тревогу с типом 3 для этого аккаунта + id1 := PgSelectAlarmAccount(conn, result[i]) + log.Println("Alarm records for account id:", result[i].mail_account_id, "found:", id1) + if id1 > 1 { + result[i].alarm_type = 3 + id = PgSelectAlarmAccount(conn, result[i]) + if id == 0 { + log.Println("Alarm insert with id", PgInsertAlarmAccount(conn, result[i], conf)) + } else { + log.Println("Alarm type",result[i].alarm_type,"with id", PgUpdateAlarm(conn, result[i], conf, id), "was updated") + } + } + + result[i].alarm_type = 1 + id2 := PgSelectAlarmRemoteIP(conn, result[i]) + log.Println("PgSelectAlarmRemoteIP with ip:",result[i].remote_ip, id2) + if id2 > 1 { + result[i].alarm_type = 4 + id = PgSelectAlarmRemoteIP(conn, result[i]) + if id == 0 { + log.Println("Alarm insert with id", PgInsertAlarmRemoteIP(conn, result[i], conf)) + } else { + log.Println("Alarm type",result[i].alarm_type,"with id", PgUpdateAlarm(conn, result[i], conf, id), "was updated") + } + } + + } + } + + defer conn.Close(context.Background()) + } +}