Obsidian/Recognition/Programing/ELK/ELK ILM 설정.md

239 lines
6.4 KiB
Markdown
Raw Normal View History

2024-06-29 03:05:20 +00:00
#ELK, #ILM
> ILM policy 설정을 통해 인덱스 자동 삭제 설정
1. Policy 생성 (Hop phase 5gb 1d, Delete 90d)
```json
PUT _ilm/policy/index-save-90days-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"set_priority": {
"priority": 100
},
"rollover": {
"max_primary_shard_size": "5gb",
"max_age": "2d"
}
}
},
"delete": {
"min_age": "90d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}
}
```
2. 템플릿 생성
- [[ElasticSearch 쿼리]] 참고하되 lifecycle 설정은 제외함(Logstash의 설정에 따라 템플릿 추가됨)
- 기존 템플릿이 있는경우 lifecycle설정이 있는지 확인해야 함.
3. Logstash 설정 (ilm_rollover_alias, ilm_policy))
```conf
# Logstash.conf
input {
beats {
port => 5044
}
}
filter {
mutate {
add_field => {
"tmp_msg_arry" => "%{[message]}"
}
}
mutate {
split => { "tmp_msg_arry" => "|" }
add_field => {
"msg_header" => "%{[tmp_msg_arry][0]}"
}
}
mutate {
split => { "msg_header" => "," }
add_field => {
"[header][recv_time]" => "%{[msg_header][0]}"
"[header][log_type]" => "%{[msg_header][1]}"
"[header][msg_type]" => "%{[msg_header][2]}"
"[header][server_se]" => "%{[msg_header][3]}"
}
}
mutate {
remove_field => [
"tmp_msg_arry", "msg_header", "@version", "agent", "ecs", "input", "_type"]
}
}
output {
if [header][msg_type] == "1"{
elasticsearch {
index => "interface_log_1_asde-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "interface_log_1_asde"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "2"{
elasticsearch {
index => "interface_log_2_ailocation-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "interface_log_2_ailocation"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "3" {
elasticsearch {
index => "replay_log_3_fusion-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_3_fusion"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "4"{
elasticsearch {
index => "interface_log_4_radar-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "interface_log_4_radar"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "5"{
elasticsearch {
index => "interface_log_5_video-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "interface_log_5_video"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "6"{
elasticsearch {
index => "replay_log_6_voice-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_6_voice"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "7"{
elasticsearch {
index => "replay_log_7_ana-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_7_ana"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "8"{
elasticsearch {
index => "replay_log_8_stand-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_8_stand"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "9"{
elasticsearch {
index => "replay_log_9_ptz-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_9_ptz"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "10"{
elasticsearch {
index => "replay_log_10_route-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_10_route"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "11"{
elasticsearch {
index => "replay_log_11_ltea-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_11_ltea"
ilm_policy => "index-save-90days-policy"
}
}
else if [header][msg_type] == "12"{
elasticsearch {
index => "replay_log_12_ptz-manual-%{+YYYY-MM-dd}"
hosts => "http://10.200.31.130:9200"
user => "elastic"
password => "changeme"
ecs_compatibility => disabled
ilm_rollover_alias => "replay_log_12_ptz.manual"
ilm_policy => "index-save-90days-policy"
}
}
}
```