Добавлено определение поля timestamp. Исключение индексов.
This commit is contained in:
Калинин Сергей Валерьевич
parent
28991e3aa6
commit
ef75a6ce81
|
|
@ -9,7 +9,7 @@
|
|||
// Require:
|
||||
// github.com/adubkov/go-zabbix
|
||||
// github.com/elastic/go-elasticsearch
|
||||
//-------------------------------------------------------------------------------
|
||||
//------------------------------------------------------------------------------
|
||||
// ZABBIX_SERVER="zabbix"
|
||||
// ZABBIX_PORT="10051"
|
||||
// ZABBIX_HOST="elastic cluster"
|
||||
|
|
@ -54,9 +54,24 @@ func StringContains(a []string, x string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
var Exclude_index_list string
|
||||
|
||||
// Проверка вхождения имени индекса (префикса) в список исключений
|
||||
func IndexPrefixExclude(index_name string) bool {
|
||||
exclude_index_name := strings.Split(Exclude_index_list, ",")
|
||||
// log.Println(Exclude_index_list)
|
||||
for _, n := range exclude_index_name {
|
||||
if strings.HasPrefix(index_name, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func EsSearch(index_name string, es_request_time_range int) int {
|
||||
var (
|
||||
r map[string]interface{}
|
||||
timestamp_field string
|
||||
)
|
||||
request_time_range := "now-" + strconv.Itoa(es_request_time_range) + "h"
|
||||
|
||||
|
|
@ -72,7 +87,8 @@ func EsSearch(index_name string, es_request_time_range int) int {
|
|||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
DialContext: (&net.Dialer{Timeout: time.Second}).DialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MaxVersion: tls.VersionTLS11,
|
||||
MinVersion: tls.VersionTLS10,
|
||||
MaxVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
|
|
@ -91,11 +107,29 @@ func EsSearch(index_name string, es_request_time_range int) int {
|
|||
if res.IsError() {
|
||||
log.Fatalf("Error: %s", res.String())
|
||||
}
|
||||
|
||||
// Checking the @timestamp field exists
|
||||
// if !EsIndexFieldExists(es, index_name, `@timestamp`) {
|
||||
// // continue
|
||||
// log.Println(EsIndexFieldExists(es, index_name, `@timestamp`), "@timestamp")
|
||||
// }
|
||||
// log.Println(EsIndexFieldExists(es, index_name, `@timestamp`), "@timestamp")
|
||||
|
||||
if EsIndexFieldExists(es, index_name, `timestamp`) {
|
||||
// continue
|
||||
timestamp_field = "timestamp"
|
||||
// log.Println(EsIndexFieldExists(es, index_name, `timestamp`), "timestamp")
|
||||
} else {
|
||||
timestamp_field = "@timestamp"
|
||||
}
|
||||
// log.Println(EsIndexFieldExists(es, index_name, `timestamp`), "timestamp")
|
||||
log.Println(index_name, timestamp_field)
|
||||
|
||||
var buf bytes.Buffer
|
||||
query := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"range": map[string]interface{}{
|
||||
"@timestamp": map[string]interface{}{
|
||||
timestamp_field: map[string]interface{}{
|
||||
// "gt": "now-1h",
|
||||
"gt": request_time_range,
|
||||
},
|
||||
|
|
@ -105,7 +139,7 @@ func EsSearch(index_name string, es_request_time_range int) int {
|
|||
},
|
||||
"size": 1,
|
||||
"sort": map[string]interface{}{
|
||||
"@timestamp": map[string]interface{}{
|
||||
timestamp_field: map[string]interface{}{
|
||||
"order": "desc",
|
||||
},
|
||||
},
|
||||
|
|
@ -236,6 +270,7 @@ func EsIndexFieldExists(es *elasticsearch.Client, index_name string, field_name
|
|||
|
||||
func EsGetIndices(es *elasticsearch.Client) []string {
|
||||
//Get indices list
|
||||
// log.Println("sjsjsjsj")
|
||||
var (
|
||||
index_request []map[string]interface{}
|
||||
es_index_list []string
|
||||
|
|
@ -250,6 +285,7 @@ func EsGetIndices(es *elasticsearch.Client) []string {
|
|||
if err != nil {
|
||||
log.Fatalf("ERROR: %s", err)
|
||||
}
|
||||
// log.Println(res)
|
||||
|
||||
if err := json.NewDecoder(res.Body).Decode(&index_request); err != nil {
|
||||
log.Fatalf("Error parsing the response body: %s", err)
|
||||
|
|
@ -260,17 +296,12 @@ func EsGetIndices(es *elasticsearch.Client) []string {
|
|||
i := m.FindStringIndex(hit["index"].(string))
|
||||
// наименование индекса
|
||||
index_name := hit["index"].(string)
|
||||
|
||||
// Checking the @timestamp field exists
|
||||
if !EsIndexFieldExists(es, index_name, `@timestamp`) {
|
||||
continue
|
||||
// fmt.Println(EsIndexFieldExists(es, index_name))
|
||||
}
|
||||
// log.Println(index_name)
|
||||
|
||||
// Итоговое наименование индекса
|
||||
var index_name_short string
|
||||
// Исключаем индексы от кибаны (.kibana-*) и обрезаем цифры (даты и версии)
|
||||
if !strings.HasPrefix(index_name, ".") {
|
||||
if !IndexPrefixExclude(index_name) {
|
||||
if i != nil {
|
||||
index_name_short = strings.TrimSpace(index_name[0:i[0]])
|
||||
} else {
|
||||
|
|
@ -279,8 +310,12 @@ func EsGetIndices(es *elasticsearch.Client) []string {
|
|||
if !StringContains(es_index_list, index_name_short) {
|
||||
es_index_list = append(es_index_list, index_name_short)
|
||||
}
|
||||
} else {
|
||||
log.Println("Index", index_name, "has name from exclude lists")
|
||||
}
|
||||
}
|
||||
log.Println(es_index_list)
|
||||
|
||||
return (es_index_list)
|
||||
}
|
||||
|
||||
|
|
@ -331,6 +366,7 @@ func main() {
|
|||
flag.StringVar(&es_index_name, "indexname", "", "Elasticsearch index name pattern, (like \"filebeat\")")
|
||||
flag.IntVar(&es_request_time_range, "timerange", 6, "Elasticsearch time range for records count into hours")
|
||||
flag.BoolVar(&zabbix_send, "zabbix-send", false, "Send metrics or discovery data into zabbix")
|
||||
flag.StringVar(&Exclude_index_list, "exclude-index", ".", "Elasticsearch index name comma-separated list for exluded, (like \"filebeat,mailindex,e.t.c\")")
|
||||
|
||||
flag.StringVar(&operation, "operation", "es-cluster-info", "Opertation type, must be:\n\tes-cluster-info - ES Cluster information (version e.t.c)\n\tes-get-indices - geting all index\n\tes-indices-discover - getting es index pattern list\n\tes-records-count - getting the number of records for a time range for all index pattern\n\tes-index-records-count - getting records count for one index (used with -indexname option")
|
||||
|
||||
|
|
@ -341,6 +377,11 @@ func main() {
|
|||
fmt.Println("Send error: make sure environment variables `ZABBIX_HOST`")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if os.Getenv("ELASTICSEARCH_URL") == "" {
|
||||
fmt.Println("Send error: make sure environment variables `ELASTICSEARCH_URL`")
|
||||
os.Exit(1)
|
||||
}
|
||||
// if es_request_time_range > 24 {
|
||||
// log.Println("Error: Time range must be a number between 1 and 24")
|
||||
// os.Exit(1)
|
||||
|
|
@ -352,7 +393,8 @@ func main() {
|
|||
ResponseHeaderTimeout: 30 * time.Second,
|
||||
DialContext: (&net.Dialer{Timeout: time.Second}).DialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MaxVersion: tls.VersionTLS11,
|
||||
MinVersion: tls.VersionTLS10,
|
||||
MaxVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
|
|
@ -384,6 +426,7 @@ func main() {
|
|||
result_index := make(map[int]json_records)
|
||||
j := int(0)
|
||||
for _, es_index_name := range EsGetIndices(es) {
|
||||
log.Println(es_index_name)
|
||||
result := EsSearch(es_index_name, es_request_time_range)
|
||||
result_index[j] = json_records{es_index_name, result}
|
||||
log.Println("ES index name:", es_index_name, "; Query time period:", es_request_time_range, "; Records count:", result)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user