From ef75a6ce8131cbae5535f0e935841a745b2595f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B0=D0=BB=D0=B8=D0=BD=D0=B8=D0=BD=20=D0=A1=D0=B5?= =?UTF-8?q?=D1=80=D0=B3=D0=B5=D0=B9=20=D0=92=D0=B0=D0=BB=D0=B5=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Tue, 10 Dec 2024 12:43:08 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=20=D0=BE=D0=BF=D1=80=D0=B5=D0=B4=D0=B5=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BB=D1=8F=20timestamp.=20?= =?UTF-8?q?=D0=98=D1=81=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=D0=BE=D0=B2.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- es-monitoring/es-monitoring.go | 69 +++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/es-monitoring/es-monitoring.go b/es-monitoring/es-monitoring.go index 74e5e8d..a6aec11 100644 --- a/es-monitoring/es-monitoring.go +++ b/es-monitoring/es-monitoring.go @@ -7,9 +7,9 @@ // Git repos: https://git.nuk-svk.ru //------------------------------------------------------------------------------- // Require: -// github.com/adubkov/go-zabbix +// github.com/adubkov/go-zabbix // github.com/elastic/go-elasticsearch -//------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // ZABBIX_SERVER="zabbix" // ZABBIX_PORT="10051" // ZABBIX_HOST="elastic cluster" @@ -54,9 +54,24 @@ func StringContains(a []string, x string) bool { return false } +var Exclude_index_list string + +// Проверка вхождения имени индекса (префикса) в список исключений +func IndexPrefixExclude(index_name string) bool { + exclude_index_name := strings.Split(Exclude_index_list, ",") + // log.Println(Exclude_index_list) + for _, n := range exclude_index_name { + if strings.HasPrefix(index_name, n) { + return true + } + } + return false +} + func EsSearch(index_name string, es_request_time_range int) int { var ( r map[string]interface{} + timestamp_field string ) request_time_range := "now-" + strconv.Itoa(es_request_time_range) + "h" @@ -72,7 +87,8 @@ func EsSearch(index_name string, es_request_time_range int) int { ResponseHeaderTimeout: 60 * time.Second, DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, TLSClientConfig: &tls.Config{ - MaxVersion: tls.VersionTLS11, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS12, InsecureSkipVerify: true, }, }, @@ -91,11 +107,29 @@ func EsSearch(index_name string, es_request_time_range int) int { if res.IsError() { log.Fatalf("Error: %s", res.String()) } + + // Checking the @timestamp field exists + // if !EsIndexFieldExists(es, index_name, `@timestamp`) { + // // continue + // log.Println(EsIndexFieldExists(es, index_name, `@timestamp`), "@timestamp") + // } + // log.Println(EsIndexFieldExists(es, index_name, `@timestamp`), "@timestamp") + + if EsIndexFieldExists(es, index_name, `timestamp`) { + // continue + timestamp_field = "timestamp" + // log.Println(EsIndexFieldExists(es, index_name, `timestamp`), "timestamp") + } else { + timestamp_field = "@timestamp" + } + // log.Println(EsIndexFieldExists(es, index_name, `timestamp`), "timestamp") + log.Println(index_name, timestamp_field) + var buf bytes.Buffer query := map[string]interface{}{ "query": map[string]interface{}{ "range": map[string]interface{}{ - "@timestamp": map[string]interface{}{ + timestamp_field: map[string]interface{}{ // "gt": "now-1h", "gt": request_time_range, }, @@ -105,7 +139,7 @@ func EsSearch(index_name string, es_request_time_range int) int { }, "size": 1, "sort": map[string]interface{}{ - "@timestamp": map[string]interface{}{ + timestamp_field: map[string]interface{}{ "order": "desc", }, }, @@ -236,6 +270,7 @@ func EsIndexFieldExists(es *elasticsearch.Client, index_name string, field_name func EsGetIndices(es *elasticsearch.Client) []string { //Get indices list + // log.Println("sjsjsjsj") var ( index_request []map[string]interface{} es_index_list []string @@ -250,6 +285,7 @@ func EsGetIndices(es *elasticsearch.Client) []string { if err != nil { log.Fatalf("ERROR: %s", err) } + // log.Println(res) if err := json.NewDecoder(res.Body).Decode(&index_request); err != nil { log.Fatalf("Error parsing the response body: %s", err) @@ -260,17 +296,12 @@ func EsGetIndices(es *elasticsearch.Client) []string { i := m.FindStringIndex(hit["index"].(string)) // наименование индекса index_name := hit["index"].(string) - - // Checking the @timestamp field exists - if !EsIndexFieldExists(es, index_name, `@timestamp`) { - continue - // fmt.Println(EsIndexFieldExists(es, index_name)) - } + // log.Println(index_name) // Итоговое наименование индекса var index_name_short string // Исключаем индексы от кибаны (.kibana-*) и обрезаем цифры (даты и версии) - if !strings.HasPrefix(index_name, ".") { + if !IndexPrefixExclude(index_name) { if i != nil { index_name_short = strings.TrimSpace(index_name[0:i[0]]) } else { @@ -279,8 +310,12 @@ func EsGetIndices(es *elasticsearch.Client) []string { if !StringContains(es_index_list, index_name_short) { es_index_list = append(es_index_list, index_name_short) } + } else { + log.Println("Index", index_name, "has name from exclude lists") } } + log.Println(es_index_list) + return (es_index_list) } @@ -331,6 +366,7 @@ func main() { flag.StringVar(&es_index_name, "indexname", "", "Elasticsearch index name pattern, (like \"filebeat\")") flag.IntVar(&es_request_time_range, "timerange", 6, "Elasticsearch time range for records count into hours") flag.BoolVar(&zabbix_send, "zabbix-send", false, "Send metrics or discovery data into zabbix") + flag.StringVar(&Exclude_index_list, "exclude-index", ".", "Elasticsearch index name comma-separated list for exluded, (like \"filebeat,mailindex,e.t.c\")") flag.StringVar(&operation, "operation", "es-cluster-info", "Opertation type, must be:\n\tes-cluster-info - ES Cluster information (version e.t.c)\n\tes-get-indices - geting all index\n\tes-indices-discover - getting es index pattern list\n\tes-records-count - getting the number of records for a time range for all index pattern\n\tes-index-records-count - getting records count for one index (used with -indexname option") @@ -341,6 +377,11 @@ func main() { fmt.Println("Send error: make sure environment variables `ZABBIX_HOST`") os.Exit(1) } + + if os.Getenv("ELASTICSEARCH_URL") == "" { + fmt.Println("Send error: make sure environment variables `ELASTICSEARCH_URL`") + os.Exit(1) + } // if es_request_time_range > 24 { // log.Println("Error: Time range must be a number between 1 and 24") // os.Exit(1) @@ -352,7 +393,8 @@ func main() { ResponseHeaderTimeout: 30 * time.Second, DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, TLSClientConfig: &tls.Config{ - MaxVersion: tls.VersionTLS11, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS13, InsecureSkipVerify: true, }, }, @@ -384,6 +426,7 @@ func main() { result_index := make(map[int]json_records) j := int(0) for _, es_index_name := range EsGetIndices(es) { + log.Println(es_index_name) result := EsSearch(es_index_name, es_request_time_range) result_index[j] = json_records{es_index_name, result} log.Println("ES index name:", es_index_name, "; Query time period:", es_request_time_range, "; Records count:", result)