Obsidian/Recognition/Programing/ELK/ELK ILM 설정.md

#ELK, #ILM

ILM policy 설정을 통해 인덱스 자동 삭제 설정

1. Policy 생성 (Hop phase 5gb 1d, Delete 90d)

PUT _ilm/policy/index-save-90days-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "set_priority": {
            "priority": 100
          },
          "rollover": {
            "max_primary_shard_size": "5gb",
            "max_age": "2d"
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {
            "delete_searchable_snapshot": true
          }
        }
      }
    }
  }
}

2. 템플릿 생성

• ElasticSearch 쿼리 참고하되 lifecycle 설정은 제외함(Logstash의 설정에 따라 템플릿 추가됨)
• 기존 템플릿이 있는경우 lifecycle설정이 있는지 확인해야 함.

3. Logstash 설정 (ilm_rollover_alias, ilm_policy))

# Logstash.conf

input {
        beats {
                port => 5044
        }
}

filter {
  mutate {
  
            add_field => {
                    "tmp_msg_arry" => "%{[message]}"
            }
            
            
            
           
  }
  
    mutate {
  
            split => { "tmp_msg_arry" => "|" }
            
            add_field => {
                    "msg_header" => "%{[tmp_msg_arry][0]}"
            }
            
           
  }
  
  mutate {
  
            split => { "msg_header" => "," }
            
            add_field => {
                    "[header][recv_time]" => "%{[msg_header][0]}"
                    "[header][log_type]" => "%{[msg_header][1]}"
                    "[header][msg_type]" => "%{[msg_header][2]}"
                    "[header][server_se]" => "%{[msg_header][3]}"
            }
            
           
  }
  
  mutate {
  
      remove_field => [
					"tmp_msg_arry", "msg_header", "@version", "agent", "ecs", "input", "_type"]
	}		

            
}


output {

   if [header][msg_type] == "1"{
       elasticsearch {
          index => "interface_log_1_asde-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "interface_log_1_asde"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "2"{
       elasticsearch {
          index => "interface_log_2_ailocation-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "interface_log_2_ailocation"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "3" {
      elasticsearch {
          index => "replay_log_3_fusion-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_3_fusion"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "4"{
       elasticsearch {
          index => "interface_log_4_radar-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "interface_log_4_radar"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "5"{
       elasticsearch {
          index => "interface_log_5_video-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "interface_log_5_video"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "6"{
       elasticsearch {
          index => "replay_log_6_voice-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_6_voice"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "7"{
       elasticsearch {
          index => "replay_log_7_ana-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_7_ana"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "8"{
       elasticsearch {
          index => "replay_log_8_stand-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_8_stand"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "9"{
       elasticsearch {
          index => "replay_log_9_ptz-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_9_ptz"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "10"{
       elasticsearch {
          index => "replay_log_10_route-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_10_route"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "11"{
       elasticsearch {
          index => "replay_log_11_ltea-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_11_ltea"
          ilm_policy => "index-save-90days-policy"
      }
   }
   else if [header][msg_type] == "12"{
       elasticsearch {
          index => "replay_log_12_ptz-manual-%{+YYYY-MM-dd}"
          hosts => "http://10.200.31.130:9200"
          user => "elastic"
      		password => "changeme"
      		ecs_compatibility => disabled
          ilm_rollover_alias => "replay_log_12_ptz.manual"
          ilm_policy => "index-save-90days-policy"
      }
   }
    
  
  
}