From dcd262ae734183e730a498d93d762f71fafc7b58 Mon Sep 17 00:00:00 2001 From: MonHun Date: Sat, 29 Jun 2024 12:05:20 +0900 Subject: [PATCH] win 0628 --- .../plugins/recent-files-obsidian/data.json | 240 +++++++++--------- Recognition/.obsidian/workspace.json | 97 ++++--- .../Programing/Docker/Docker 명령어.md | 3 + Recognition/Programing/ELK/ELK ILM 설정.md | 239 +++++++++++++++++ .../Programing/ELK/ELK 백업 및 복구.md | 58 +++++ Recognition/Programing/ELK/ELK 오류.md | 28 ++ .../ELK/ELK 폐쇄망 설치(New).md | 10 + .../Programing/ELK/ElasticSearch 쿼리.md | 114 ++++++++- .../Programing/Tibero/무제 파일.md | 13 + .../Work Related/1.업무메모/2024-04-26.md | 8 + .../9.기타/Filebeat 재배포.md | 14 + .../프로젝트/스마트계류장/TODO-0.md | 4 + 12 files changed, 667 insertions(+), 161 deletions(-) create mode 100644 Recognition/Programing/ELK/ELK ILM 설정.md create mode 100644 Recognition/Programing/ELK/ELK 백업 및 복구.md create mode 100644 Recognition/Programing/ELK/ELK 오류.md create mode 100644 Recognition/Programing/Tibero/무제 파일.md diff --git a/Recognition/.obsidian/plugins/recent-files-obsidian/data.json b/Recognition/.obsidian/plugins/recent-files-obsidian/data.json index df117a7..304fa8d 100644 --- a/Recognition/.obsidian/plugins/recent-files-obsidian/data.json +++ b/Recognition/.obsidian/plugins/recent-files-obsidian/data.json @@ -1,5 +1,125 @@ { "recentFiles": [ + { + "basename": "Filebeat 재배포", + "path": "Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md" + }, + { + "basename": "TODO-0", + "path": "Work Related/프로젝트/스마트계류장/TODO-0.md" + }, + { + "basename": "EAI 연계 데이터 복구", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 연계 데이터 복구.md" + }, + { + "basename": "EAI 데이터 초기화", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 데이터 초기화.md" + }, + { + "basename": "EAI 프로시져 테스트", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 프로시져 테스트.md" + }, + { + "basename": "SACP_AC_MASTER 갱신", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/SACP_AC_MASTER 갱신.md" + }, + { + "basename": "GAM_LIST 데이터 일괄 insert", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/GAM_LIST 데이터 일괄 insert.md" + }, + { + "basename": "EAI-Data Item 추출작업", + "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI-Data Item 추출작업.md" + }, + { + "basename": "Java 날짜 관련", + "path": "Programing/Java/Java 날짜 관련.md" + }, + { + "basename": "EAI Server 셋팅", + "path": "Work Related/프로젝트/스마트계류장/외부연계 관련/EAI/EAI Server 셋팅.md" + }, + { + "basename": "ElasticSearch 쿼리", + "path": "Programing/ELK/ElasticSearch 쿼리.md" + }, + { + "basename": "ELK 오류", + "path": "Programing/ELK/ELK 오류.md" + }, + { + "basename": "ElasticSearch 인덱스 통계", + "path": "Programing/ELK/ElasticSearch 인덱스 통계.md" + }, + { + "basename": "ELK 백업 및 복구", + "path": "Programing/ELK/ELK 백업 및 복구.md" + }, + { + "basename": "ELK 폐쇄망 설치(New)", + "path": "Programing/ELK/ELK 폐쇄망 설치(New).md" + }, + { + "basename": "무제 파일", + "path": "Programing/Tibero/무제 파일.md" + }, + { + "basename": "query DSL", + "path": "Programing/ELK/query DSL.md" + }, + { + "basename": "2024-04-26", + "path": "Work Related/1.업무메모/2024-04-26.md" + }, + { + "basename": "2024-04-25", + "path": "Work Related/1.업무메모/2024-04-25.md" + }, + { + "basename": "2024-04-22", + "path": "Work Related/1.업무메모/2024-04-22.md" + }, + { + "basename": "2024-03-25", + "path": "Work Related/1.업무메모/2024-03-25.md" + }, + { + "basename": "프로세스 확인 및 강제 종료", + "path": "Programing/Linux/Ubuntu/프로세스 확인 및 강제 종료.md" + }, + { + "basename": "ELK ILM 설정", + "path": "Programing/ELK/ELK ILM 설정.md" + }, + { + "basename": "Cluster", + "path": "Programing/ELK/Cluster.md" + }, + { + "basename": "사용Port", + "path": "Programing/ELK/사용Port.md" + }, + { + "basename": "무제 파일", + "path": "Programing/Docker/무제 파일.md" + }, + { + "basename": "ElasticSearch 검색 관련", + "path": "Programing/ELK/ElasticSearch 검색 관련.md" + }, + { + "basename": "Elastic Search + Spring Boot에서 검색 로직", + "path": "Programing/ELK/Elastic Search + Spring Boot에서 검색 로직.md" + }, + { + "basename": "Docker 명령어", + "path": "Programing/Docker/Docker 명령어.md" + }, + { + "basename": "Docker Compose", + "path": "Programing/Docker/Docker Compose.md" + }, { "basename": "자주참고하는 노트", "path": "0.바로가기/자주참고하는 노트.md" @@ -8,10 +128,6 @@ "basename": "수협본회 서버 작업", "path": "Work Related/프로젝트/D-MHFH/유지보수/수협본회 서버 작업.md" }, - { - "basename": "TODO-0", - "path": "Work Related/프로젝트/스마트계류장/TODO-0.md" - }, { "basename": "공유문서", "path": "Work Related/프로젝트/스마트계류장/0.관련정보/공유문서.md" @@ -24,10 +140,6 @@ "basename": "관리페이지 테스트 쿼리", "path": "Work Related/프로젝트/스마트계류장/1.데이터베이스/관리페이지 테스트 쿼리.md" }, - { - "basename": "Java 날짜 관련", - "path": "Programing/Java/Java 날짜 관련.md" - }, { "basename": "Lock 조회", "path": "Programing/Tibero/Lock 조회.md" @@ -87,118 +199,6 @@ { "basename": "약어표", "path": "Work Related/프로젝트/스마트계류장/항공용어/약어표.md" - }, - { - "basename": "용어정리", - "path": "Work Related/프로젝트/스마트계류장/항공용어/용어정리.md" - }, - { - "basename": "Jenkins script 모음", - "path": "Programing/Jenkins/Jenkins script 모음.md" - }, - { - "basename": "Jenkins 설치", - "path": "Programing/Jenkins/Jenkins 설치.md" - }, - { - "basename": "특정 포트 임시 오픈", - "path": "Programing/Linux/Ubuntu/특정 포트 임시 오픈.md" - }, - { - "basename": "특정포트 허용", - "path": "Programing/Linux/RHEL9/특정포트 허용.md" - }, - { - "basename": "무제노트", - "path": "Programing/Network/무제노트.md" - }, - { - "basename": "스키마 생성", - "path": "Programing/Tibero/스키마 생성.md" - }, - { - "basename": "권한(Grant)", - "path": "Programing/Tibero/권한(Grant).md" - }, - { - "basename": "Tibero 이중화 구성시 서비스 실행", - "path": "Programing/Tibero/Tibero 이중화 구성시 서비스 실행.md" - }, - { - "basename": "방문증 신청 문구", - "path": "Work Related/프로젝트/스마트계류장/9.기타/방문증 신청 문구.md" - }, - { - "basename": "Kafka ELK로 모니터링", - "path": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md" - }, - { - "basename": "기동 및 종료", - "path": "Programing/Tibero/기동 및 종료.md" - }, - { - "basename": "TPR 확인", - "path": "Programing/Tibero/TPR 확인.md" - }, - { - "basename": "2022-06-15", - "path": "Work Related/1.업무메모/2022.06/2022-06-15.md" - }, - { - "basename": "VSCode 단축키", - "path": "Tools/VSCode/VSCode 단축키.md" - }, - { - "basename": "2.단축키", - "path": "Tools/Obsidan/2.단축키.md" - }, - { - "basename": "JAVA Timezone ID", - "path": "Programing/Java/JAVA Timezone ID.md" - }, - { - "basename": "DB 통계 관련", - "path": "Programing/Tibero/DB 통계 관련.md" - }, - { - "basename": "2022-03-14", - "path": "Work Related/1.업무메모/2022.03/2022-03-14.md" - }, - { - "basename": "2022-03-04", - "path": "Work Related/1.업무메모/2022.03/2022-03-04.md" - }, - { - "basename": "2022-05-10", - "path": "Work Related/1.업무메모/2022.05/2022-05-10.md" - }, - { - "basename": "Loop문", - "path": "Programing/Oracle/Loop문.md" - }, - { - "basename": "ElasticSearch 인덱스 통계", - "path": "Programing/ELK/ElasticSearch 인덱스 통계.md" - }, - { - "basename": "ElasticSearch 쿼리", - "path": "Programing/ELK/ElasticSearch 쿼리.md" - }, - { - "basename": "무제 파일 1", - "path": "무제 파일 1.canvas" - }, - { - "basename": "HAProxy HA (High Availability) 구성하기", - "path": "Programing/Linux/RHEL9/HAProxy HA (High Availability) 구성하기.md" - }, - { - "basename": "2024-04-26", - "path": "Work Related/1.업무메모/2024-04-26.md" - }, - { - "basename": "query DSL", - "path": "Programing/ELK/query DSL.md" } ], "omittedPaths": [], diff --git a/Recognition/.obsidian/workspace.json b/Recognition/.obsidian/workspace.json index c2de8e5..00ce820 100644 --- a/Recognition/.obsidian/workspace.json +++ b/Recognition/.obsidian/workspace.json @@ -4,16 +4,16 @@ "type": "split", "children": [ { - "id": "3ba27af2a1fe0e82", + "id": "317a23199b7d79cd", "type": "tabs", "children": [ { - "id": "5dcb9d7813128e94", + "id": "2d9948953b158174", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "0.바로가기/자주참고하는 노트.md", + "file": "Programing/ELK/ELK 오류.md", "mode": "source", "backlinks": true, "source": false @@ -21,19 +21,46 @@ } }, { - "id": "4bf4b373ebb882c9", + "id": "164c02dc8a39e1af", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Work Related/프로젝트/스마트계류장/0.관련정보/공유문서.md", + "file": "Programing/ELK/ELK 백업 및 복구.md", + "mode": "source", + "backlinks": true, + "source": false + } + } + }, + { + "id": "b3db94b54b07553d", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Work Related/프로젝트/스마트계류장/외부연계 관련/EAI/EAI Server 셋팅.md", + "mode": "source", + "backlinks": true, + "source": false + } + } + }, + { + "id": "4241f12287aaedc6", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md", "mode": "source", "backlinks": true, "source": false } } } - ] + ], + "currentTab": 3 } ], "direction": "vertical" @@ -62,7 +89,7 @@ "state": { "type": "search", "state": { - "query": "timestamp", + "query": "filebeat", "matchingCase": false, "explainSearch": false, "collapseAll": true, @@ -116,7 +143,7 @@ "state": { "type": "backlink", "state": { - "file": "0.바로가기/자주참고하는 노트.md", + "file": "Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md", "collapseAll": false, "extraContext": false, "sortOrder": "alphabetical", @@ -133,7 +160,7 @@ "state": { "type": "outgoing-link", "state": { - "file": "0.바로가기/자주참고하는 노트.md", + "file": "Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md", "linksCollapsed": false, "unlinkedCollapsed": true } @@ -156,7 +183,7 @@ "state": { "type": "outline", "state": { - "file": "0.바로가기/자주참고하는 노트.md" + "file": "Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md" } } }, @@ -198,34 +225,34 @@ "command-palette:명령어 팔레트 열기": false } }, - "active": "5dcb9d7813128e94", + "active": "4241f12287aaedc6", "lastOpenFiles": [ - "Work Related/프로젝트/D-MHFH/유지보수/수협본회 서버 작업.md", "Work Related/프로젝트/스마트계류장/TODO-0.md", - "Work Related/프로젝트/스마트계류장/0.관련정보/공유문서.md", - "Work Related/프로젝트/스마트계류장/TODO.md", - "Programing/Tibero/Tibero 세션 관련.md", - "Work Related/프로젝트/스마트계류장/1.데이터베이스/관리페이지 테스트 쿼리.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 연계 데이터 복구.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 데이터 초기화.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI 프로시져 테스트.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/SACP_AC_MASTER 갱신.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/GAM_LIST 데이터 일괄 insert.md", + "Work Related/프로젝트/스마트계류장/1.데이터베이스/EAI-Data Item 추출작업.md", "Programing/Java/Java 날짜 관련.md", - "Programing/Tibero/Lock 조회.md", - "Work Related/3.주간계획/스마트계류장/2023.10-4.md", - "Work Related/3.주간계획/스마트계류장/2023.10-3.md", - "Programing/0.Root/용어.md", - "Work Related/1.업무메모/2022.07/2022-07-05.md", - "Programing/Tibero/Tibero Export & Import.md", - "Work Related/프로젝트/스마트계류장/통계관련.md", - "Programing/Linux/Linux 명령어/리눅스 특정 폴더, 파일 용량 확인.md", - "Programing/Linux/Unix/스케줄려 생성.md", - "Programing/Tibero/archive 삭제 스크립트.md", - "Programing/Oracle/데이터 타입.md", - "Programing/Tibero/테이블 Archive 로그 설정 변경.md", - "Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md", - "Programing/Tibero/function Table.md", - "Work Related/프로젝트/스마트계류장/항공용어/약어표.md", - "Work Related/프로젝트/스마트계류장/항공용어/용어정리.md", - "Programing/Jenkins/Jenkins script 모음.md", - "Programing/Jenkins/Jenkins 설치.md", - "Programing/Linux/Ubuntu/특정 포트 임시 오픈.md", + "Work Related/프로젝트/스마트계류장/외부연계 관련/EAI/EAI Server 셋팅.md", + "Programing/ELK/ElasticSearch 쿼리.md", + "Programing/ELK/ELK 오류.md", + "Programing/ELK/ElasticSearch 인덱스 통계.md", + "Programing/ELK/ELK 폐쇄망 설치(New).md", + "Programing/ELK/ELK 백업 및 복구.md", + "Programing/Tibero/무제 파일.md", + "Programing/ELK/query DSL.md", + "Work Related/1.업무메모/2024-04-26.md", + "Work Related/1.업무메모/2024-04-25.md", + "Work Related/1.업무메모/2024-04-22.md", + "Work Related/1.업무메모/2024-03-25.md", + "Programing/Linux/Ubuntu/프로세스 확인 및 강제 종료.md", + "Programing/ELK/ELK ILM 설정.md", + "Programing/ELK/Cluster.md", + "Programing/ELK/사용Port.md", + "Programing/Docker/무제 파일.md", + "Programing/ELK/ElasticSearch 검색 관련.md", "zzz.resources/Pasted image 20240514161803.png", "zzz.resources/Pasted image 20240514161647.png", "무제 파일 1.canvas", diff --git a/Recognition/Programing/Docker/Docker 명령어.md b/Recognition/Programing/Docker/Docker 명령어.md index 950a387..a620abd 100644 --- a/Recognition/Programing/Docker/Docker 명령어.md +++ b/Recognition/Programing/Docker/Docker 명령어.md @@ -65,6 +65,9 @@ - 컨테이너 로그 보기 > docker logs es01 +> docker logs --tail 10 +> - t : 타임스템프 추가 +> - f : 스트림 https://arisu1000.tistory.com/27786 diff --git a/Recognition/Programing/ELK/ELK ILM 설정.md b/Recognition/Programing/ELK/ELK ILM 설정.md new file mode 100644 index 0000000..7854bbf --- /dev/null +++ b/Recognition/Programing/ELK/ELK ILM 설정.md @@ -0,0 +1,239 @@ +#ELK, #ILM + +> ILM policy 설정을 통해 인덱스 자동 삭제 설정 + +1. Policy 생성 (Hop phase 5gb 1d, Delete 90d) +```json + +PUT _ilm/policy/index-save-90days-policy +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "set_priority": { + "priority": 100 + }, + "rollover": { + "max_primary_shard_size": "5gb", + "max_age": "2d" + } + } + }, + "delete": { + "min_age": "90d", + "actions": { + "delete": { + "delete_searchable_snapshot": true + } + } + } + } + } +} + +``` + +2. 템플릿 생성 +- [[ElasticSearch 쿼리]] 참고하되 lifecycle 설정은 제외함(Logstash의 설정에 따라 템플릿 추가됨) +- 기존 템플릿이 있는경우 lifecycle설정이 있는지 확인해야 함. + +3. Logstash 설정 (ilm_rollover_alias, ilm_policy)) +```conf + +# Logstash.conf + +input { + beats { + port => 5044 + } +} + +filter { + mutate { + + add_field => { + "tmp_msg_arry" => "%{[message]}" + } + + + + + } + + mutate { + + split => { "tmp_msg_arry" => "|" } + + add_field => { + "msg_header" => "%{[tmp_msg_arry][0]}" + } + + + } + + mutate { + + split => { "msg_header" => "," } + + add_field => { + "[header][recv_time]" => "%{[msg_header][0]}" + "[header][log_type]" => "%{[msg_header][1]}" + "[header][msg_type]" => "%{[msg_header][2]}" + "[header][server_se]" => "%{[msg_header][3]}" + } + + + } + + mutate { + + remove_field => [ + "tmp_msg_arry", "msg_header", "@version", "agent", "ecs", "input", "_type"] + } + + +} + + +output { + + if [header][msg_type] == "1"{ + elasticsearch { + index => "interface_log_1_asde-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "interface_log_1_asde" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "2"{ + elasticsearch { + index => "interface_log_2_ailocation-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "interface_log_2_ailocation" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "3" { + elasticsearch { + index => "replay_log_3_fusion-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_3_fusion" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "4"{ + elasticsearch { + index => "interface_log_4_radar-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "interface_log_4_radar" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "5"{ + elasticsearch { + index => "interface_log_5_video-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "interface_log_5_video" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "6"{ + elasticsearch { + index => "replay_log_6_voice-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_6_voice" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "7"{ + elasticsearch { + index => "replay_log_7_ana-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_7_ana" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "8"{ + elasticsearch { + index => "replay_log_8_stand-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_8_stand" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "9"{ + elasticsearch { + index => "replay_log_9_ptz-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_9_ptz" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "10"{ + elasticsearch { + index => "replay_log_10_route-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_10_route" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "11"{ + elasticsearch { + index => "replay_log_11_ltea-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_11_ltea" + ilm_policy => "index-save-90days-policy" + } + } + else if [header][msg_type] == "12"{ + elasticsearch { + index => "replay_log_12_ptz-manual-%{+YYYY-MM-dd}" + hosts => "http://10.200.31.130:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + ilm_rollover_alias => "replay_log_12_ptz.manual" + ilm_policy => "index-save-90days-policy" + } + } + + + +} + +``` \ No newline at end of file diff --git a/Recognition/Programing/ELK/ELK 백업 및 복구.md b/Recognition/Programing/ELK/ELK 백업 및 복구.md new file mode 100644 index 0000000..2baa928 --- /dev/null +++ b/Recognition/Programing/ELK/ELK 백업 및 복구.md @@ -0,0 +1,58 @@ +https://ryeon9445.com/develop/1-elasitcsearch-snapshot/ + + +```shell + + +# elasticsearch.yml 편집 +path.repo: ["/usr/share/elasticsearch/backup"] + +# docker-compose 볼륨 추가 +volumes: + - /data/elk_data/backup:/usr/share/elasticsearch/backup + +# Kibana에서 스냅샷 레파지토리 생성 +PUT /_snapshot/backup +{ + "type": "fs", + "settings": { + "compress": true, + "location": "/usr/share/elasticsearch/backup" + } +} + +# 레파지토리 생성 확인 +GET _snapshot + +# 레파지토리 소유 권한 변경 +docker exec -it es01 bash +cd /usr/share/elasticsearch +chown -R elasticsearch:elasticsearch ./backup + +# 스냅샷 생성 +PUT /_snapshot/backup/backup_20240625?wait_for_completion=true +{ + "ignore_unavailable": true, + "include_global_state": true +} + +## 특정인덱스 지정 하련는 경우 : "indices": ${인덱스명} 추가 + +------------------------------------- +kibana Snapshot and Restore -> policy 생성 +Name : log_backup +Snapshot name : + + +------------------------------------- + +# 스냅샷 삭제 +DELETE /_snapshot/backup/test-2018.12.06 + + +# 레파지토리 삭제 +DELETE /_snapshot/backup + + + +``` \ No newline at end of file diff --git a/Recognition/Programing/ELK/ELK 오류.md b/Recognition/Programing/ELK/ELK 오류.md new file mode 100644 index 0000000..629520b --- /dev/null +++ b/Recognition/Programing/ELK/ELK 오류.md @@ -0,0 +1,28 @@ +#ELK + +- filebeat 재시작후 Logstash 로그 +> ... message=>"No Available connections" ... +> [2024-06-21T05:50:13,531][ERROR][logstash.outputs.elasticsearch][main][3394228cfe781c7f5045d8b255bb26c9e991ecaafd2f2ab6a3b19b0b8be87572] Attempted to send a bulk request but there are no living connections in the pool (perhaps Elasticsearch is unreachable or down?) {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError, :will_retry_in_seconds=>16} + +```yml +# doker-compose env에 아래 설정 추가 + +user: root + +- xpack.monitoring.enabled=false + + +``` + +- "Caused by: org.elasticsearch.action.search.SearchPhaseExecutionException: Search rejected due to missing shards + + + +--- + +```txt + + "level": "ERROR", "component": "o.e.x.d.l.DeprecationIndexingComponent", "cluster.name": "docker-cluster", "node.name": "es01-node", "message": "Bulk write of deprecation logs encountered some failures: [[eVR2TpABDRy1v2BxD6Kv UnavailableShardsException[[.ds-.logs-deprecation.elasticsearch-default-2024.06.19-000005][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.ds-.logs-deprecation.elasticsearch-default-2024.06.19-000005][0]] containing + +``` + diff --git a/Recognition/Programing/ELK/ELK 폐쇄망 설치(New).md b/Recognition/Programing/ELK/ELK 폐쇄망 설치(New).md index 3088e0d..ebdf3b9 100644 --- a/Recognition/Programing/ELK/ELK 폐쇄망 설치(New).md +++ b/Recognition/Programing/ELK/ELK 폐쇄망 설치(New).md @@ -20,6 +20,16 @@ docker exec -it es01 bash docker exec -it --user root es01 bash +# Kibana DevTool에서 샤드수 제한 늘리기 +PUT /_cluster/settings +{ + "persistent": { + "cluster.max_shards_per_node": "5000" + } +} + +https://velog.io/@diane_at_work/Elasticsearch-auto-rollover-%EC%A0%81%EC%9A%A9 + ``` diff --git a/Recognition/Programing/ELK/ElasticSearch 쿼리.md b/Recognition/Programing/ELK/ElasticSearch 쿼리.md index 0218dce..846124d 100644 --- a/Recognition/Programing/ELK/ElasticSearch 쿼리.md +++ b/Recognition/Programing/ELK/ElasticSearch 쿼리.md @@ -1,13 +1,23 @@ ## 템플릿 생성 ```Json -PUT _template/replay_log_template +PUT _template/replay_log_3_fusion { + "order" : 1, + "version" : 60001, "index_patterns": [ - "replay_log_*" // 적용할 인덱스 패턴 + "replay_log_3_fusion-*" ], "settings": { - "max_result_window" : 500000 // 50만개까지 한꺼번에 조회 가능하도록 설정 + "index": { + "lifecycle": { + "name": "index-save-90days-policy", + "rollover_alias": "replay_log_3_fusion" + } + }, + "max_result_window" : 500000, + "number_of_shards": "1", + "number_of_replicas": "0" }, "mappings": { "properties": { @@ -21,7 +31,7 @@ PUT _template/replay_log_template "fields": { "keyword": { "type": "keyword", - "ignore_above": 1 // 1 자리 이외의 데이터는 검색에서 무시 + "ignore_above": 1 } } }, @@ -30,6 +40,7 @@ PUT _template/replay_log_template "fields": { "keyword": { "type": "keyword", + "ignore_above": 1 } } }, @@ -38,7 +49,7 @@ PUT _template/replay_log_template "fields": { "keyword": { "type": "keyword", - "eager_global_ordinals": true // 집계에 많이 사용되는 필드를 캐시 + "eager_global_ordinals": true } } }, @@ -131,6 +142,15 @@ PUT replay_log_7_ana-2024-02-27_re } } } + + +PUT rolling-test-000001 +{ + "aliases": { + "rolling-write-index": { "is_write_index": true } + } +} + ``` ## 인덱스, 템플릿 삭제 ```json @@ -226,6 +246,49 @@ POST replay_log_*_*/_search ``` +## Alias 추가/삭제 +```json +// Alias 추가 +POST /_aliases?pretty +{ + "actions": [ + { + "add": { + "index": "replay_log_9_ptz-2024-03-12", + "alias": "replay_log", + "is_write_index": true + } + }, + { + "add": { + "index": "replay_log_9_ptz-2024-03-13", + "alias": "replay_log", + "is_write_index": true + } + } + ] +} + +// Alias 삭제 +POST /_aliases?pretty +{ + "actions": [ + { + "remove": { + "index": "replay_log_9_ptz-2024-03-12", + "alias": "replay_log" + } + }, + { + "remove": { + "index": "replay_log_9_ptz-2024-03-13", + "alias": "replay_log" + } + } + ] +} +``` + ## 총 데이터 건수 가져오기 ```json @@ -254,7 +317,7 @@ POST test_index/_bulk ## 인덱스별 용량 ```json -GET /_cat/indices/st*?v&s=index +GET /_cat/indices/replay*?v&s=index // S : (Optional, string) 응답을 정렬하는 데 사용되는 열 이름 또는 열 별칭의 쉼표로 구분된 목록입니다. // V : (Optional, boolean) true인 경우 응답에 열 제목이 포함됩니다. 기본값은 false입니다. @@ -282,6 +345,45 @@ GET _cat/allocation?v // Optimize curl -XPOST 'http://localhost:9200/twitter/_optimize' +// 클러스터 정보 조회 +http://10.200.31.130:9200/_cluster/settings?include_defaults&flat_settings +(max_shards_per_node) + +// alias 정보 조회 +GET _alias/replay_log* + +GET /_cat/aliases/?v=true&s=index + + +// 인덱스 Policy 정보 조회 +GET interface_log*/_ilm/explain + +// 특정 Policy 상세정보 +GET _ilm/policy/index-save-90days-policy + +// Policy 일괄 적용 +PUT /replay_log*/_settings +{ + "index": { + "lifecycle": { + "name": "index-save-90days-policy", + "rollover_alias": "replay_log" + } + } +} + +// 템플릿 정보 확인 +GET _cat/templates?v&s=name + +// 특정 템플릿 상세 정보 +GET _template/replay_log_9_ptz + + +// template 삭제 +DELETE _template/travel-log-template + +// 샤드 리스트 확인 +GET /_cat/shards?h=index,shard,prirep,state,unassigned.reason ``` \ No newline at end of file diff --git a/Recognition/Programing/Tibero/무제 파일.md b/Recognition/Programing/Tibero/무제 파일.md new file mode 100644 index 0000000..b612ecc --- /dev/null +++ b/Recognition/Programing/Tibero/무제 파일.md @@ -0,0 +1,13 @@ +``` cmd + +tbexport -u {username} -p {password} -S {service_name} -tbs {tablespace_name} -d {export_directory} + +tbexport -u myuser -p mypassword -S mydb -tbs mytablespace -d /backup/exports + + +tbimport -u {username} -p {password} -S {service_name} -tbs {tablespace_name} -d {import_directory} + + +tbexport -u myuser -p mypassword -S mydb -tbs mytablespace -d /backup/exports + +``` \ No newline at end of file diff --git a/Recognition/Work Related/1.업무메모/2024-04-26.md b/Recognition/Work Related/1.업무메모/2024-04-26.md index d7a311d..d0a555d 100644 --- a/Recognition/Work Related/1.업무메모/2024-04-26.md +++ b/Recognition/Work Related/1.업무메모/2024-04-26.md @@ -10,3 +10,11 @@ cctv제어내용 (수동조작) HotSpot 교통량 ? +```table +``` + + +| | | | | +| --- | --- | --- | --- | +| | | | | +cc \ No newline at end of file diff --git a/Recognition/Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md b/Recognition/Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md index 6753937..218b11d 100644 --- a/Recognition/Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md +++ b/Recognition/Work Related/프로젝트/스마트계류장/9.기타/Filebeat 재배포.md @@ -36,4 +36,18 @@ tss_ai_location cd /home/gmt/app/filebeat/ && ./start.sh + + +--------------------------------------------------------- +json 포멧 전송 필요 시 아래 추가 + +- type: log + enabled: true + paths: + - /home/gmt/app/EyeGW_EaiIf/logs/STATS/stats.log + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true + json.expand_keys: true + ``` \ No newline at end of file diff --git a/Recognition/Work Related/프로젝트/스마트계류장/TODO-0.md b/Recognition/Work Related/프로젝트/스마트계류장/TODO-0.md index 3d27798..afa6b21 100644 --- a/Recognition/Work Related/프로젝트/스마트계류장/TODO-0.md +++ b/Recognition/Work Related/프로젝트/스마트계류장/TODO-0.md @@ -53,3 +53,7 @@ - 서버 테스트 사항 정리 - [ ] aphq 정리 + + +- [ ] STATS 모듈 1초주기 상태전송 +- [ ] EAI- ALL 프로시져 적용 \ No newline at end of file