From 2cba5c2917b470c73bc074941b376ee4f2d00ec7 Mon Sep 17 00:00:00 2001 From: MonHun Date: Thu, 9 May 2024 18:01:30 +0900 Subject: [PATCH] win 0509 --- .../plugins/recent-files-obsidian/data.json | 56 ++-- Recognition/.obsidian/workspace.json | 62 +--- .../ELK/ElasticSearch 인덱스 통계.md | 274 ++++++++++++++++++ .../Programing/ELK/ElasticSearch 쿼리.md | 22 ++ Recognition/Programing/ELK/무제 파일.md | 0 .../0.관련정보/미들웨어 API.md | 2 +- Recognition/무제 파일 1.canvas | 1 + 7 files changed, 342 insertions(+), 75 deletions(-) create mode 100644 Recognition/Programing/ELK/ElasticSearch 인덱스 통계.md delete mode 100644 Recognition/Programing/ELK/무제 파일.md create mode 100644 Recognition/무제 파일 1.canvas diff --git a/Recognition/.obsidian/plugins/recent-files-obsidian/data.json b/Recognition/.obsidian/plugins/recent-files-obsidian/data.json index c487d22..390a806 100644 --- a/Recognition/.obsidian/plugins/recent-files-obsidian/data.json +++ b/Recognition/.obsidian/plugins/recent-files-obsidian/data.json @@ -1,17 +1,41 @@ { "recentFiles": [ + { + "basename": "미들웨어 API", + "path": "Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md" + }, + { + "basename": "ElasticSearch 인덱스 통계", + "path": "Programing/ELK/ElasticSearch 인덱스 통계.md" + }, { "basename": "Kafka ELK로 모니터링", "path": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md" }, - { - "basename": "query DSL", - "path": "Programing/ELK/query DSL.md" - }, { "basename": "ElasticSearch 쿼리", "path": "Programing/ELK/ElasticSearch 쿼리.md" }, + { + "basename": "무제 파일 1", + "path": "무제 파일 1.canvas" + }, + { + "basename": "HAProxy HA (High Availability) 구성하기", + "path": "Programing/Linux/RHEL9/HAProxy HA (High Availability) 구성하기.md" + }, + { + "basename": "2024-04-26", + "path": "Work Related/1.업무메모/2024-04-26.md" + }, + { + "basename": "2024-04-30", + "path": "Work Related/1.업무메모/2024-04-30.md" + }, + { + "basename": "query DSL", + "path": "Programing/ELK/query DSL.md" + }, { "basename": "REST API Naming", "path": "Programing/z.기타/REST API Naming.md" @@ -20,10 +44,6 @@ "basename": "ElasticSearch 검색 관련", "path": "Programing/ELK/ElasticSearch 검색 관련.md" }, - { - "basename": "무제 파일", - "path": "Programing/ELK/무제 파일.md" - }, { "basename": "Cluster", "path": "Programing/ELK/Cluster.md" @@ -32,10 +52,6 @@ "basename": "Elastic Search + Spring Boot에서 검색 로직", "path": "Programing/ELK/Elastic Search + Spring Boot에서 검색 로직.md" }, - { - "basename": "HAProxy HA (High Availability) 구성하기", - "path": "Programing/Linux/RHEL9/HAProxy HA (High Availability) 구성하기.md" - }, { "basename": "PassWD", "path": "Zettelkasten/PassWD.md" @@ -52,14 +68,6 @@ "basename": "Home Network", "path": "Zettelkasten/Home Network.md" }, - { - "basename": "2024-04-30", - "path": "Work Related/1.업무메모/2024-04-30.md" - }, - { - "basename": "2024-04-26", - "path": "Work Related/1.업무메모/2024-04-26.md" - }, { "basename": "TODO", "path": "Work Related/프로젝트/스마트계류장/TODO.md" @@ -172,10 +180,6 @@ "basename": "Kafka-ELK 테스트", "path": "Work Related/프로젝트/스마트계류장/0.관련정보/Kafka-ELK 테스트.md" }, - { - "basename": "미들웨어 API", - "path": "Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md" - }, { "basename": "0.Learnig List", "path": "Programing/0.Learnig List.md" @@ -195,10 +199,6 @@ { "basename": "CMD 명령어", "path": "Programing/Windows/CMD 명령어.md" - }, - { - "basename": "2023.10-3", - "path": "Work Related/3.주간계획/스마트계류장/2023.10-3.md" } ], "omittedPaths": [], diff --git a/Recognition/.obsidian/workspace.json b/Recognition/.obsidian/workspace.json index cac34d1..c007b92 100644 --- a/Recognition/.obsidian/workspace.json +++ b/Recognition/.obsidian/workspace.json @@ -4,42 +4,16 @@ "type": "split", "children": [ { - "id": "3929936619661553", + "id": "f5238c176cbbbfbf", "type": "tabs", "children": [ { - "id": "001a57375ba464a4", + "id": "db38be15ffab188d", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", - "mode": "source", - "backlinks": true, - "source": false - } - } - }, - { - "id": "12c1412d3ddc7287", - "type": "leaf", - "state": { - "type": "markdown", - "state": { - "file": "Work Related/1.업무메모/2024-04-30.md", - "mode": "source", - "backlinks": true, - "source": false - } - } - }, - { - "id": "68d1060d33812d56", - "type": "leaf", - "state": { - "type": "markdown", - "state": { - "file": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", + "file": "Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md", "mode": "source", "backlinks": true, "source": false @@ -75,7 +49,7 @@ "state": { "type": "search", "state": { - "query": "metricbeat", + "query": "getSql", "matchingCase": false, "explainSearch": false, "collapseAll": true, @@ -108,7 +82,8 @@ "state": {} } } - ] + ], + "currentTab": 1 } ], "direction": "horizontal", @@ -128,7 +103,6 @@ "state": { "type": "backlink", "state": { - "file": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", "collapseAll": false, "extraContext": false, "sortOrder": "alphabetical", @@ -145,7 +119,6 @@ "state": { "type": "outgoing-link", "state": { - "file": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", "linksCollapsed": false, "unlinkedCollapsed": true } @@ -167,9 +140,7 @@ "type": "leaf", "state": { "type": "outline", - "state": { - "file": "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md" - } + "state": {} } }, { @@ -210,23 +181,24 @@ "command-palette:명령어 팔레트 열기": false } }, - "active": "001a57375ba464a4", + "active": "12ad47a141496a9a", "lastOpenFiles": [ - "Programing/ELK/query DSL.md", + "Programing/ELK/ElasticSearch 인덱스 통계.md", + "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", "Programing/ELK/ElasticSearch 쿼리.md", + "무제 파일 1.canvas", + "Programing/Linux/RHEL9/HAProxy HA (High Availability) 구성하기.md", + "Work Related/1.업무메모/2024-04-26.md", + "Work Related/1.업무메모/2024-04-30.md", + "Programing/ELK/query DSL.md", "Programing/z.기타/REST API Naming.md", "Programing/ELK/ElasticSearch 검색 관련.md", - "Programing/ELK/무제 파일.md", "Programing/ELK/Cluster.md", "Programing/ELK/Elastic Search + Spring Boot에서 검색 로직.md", - "Programing/Kafka(AMQP)/Kafka ELK로 모니터링.md", - "Programing/Linux/RHEL9/HAProxy HA (High Availability) 구성하기.md", "Zettelkasten/PassWD.md", "Programing/Linux/RHEL9/프로세스 이름으로 확인.md", "Programing/Jenkins/Jenkins 설치.md", "Zettelkasten/Home Network.md", - "Work Related/1.업무메모/2024-04-30.md", - "Work Related/1.업무메모/2024-04-26.md", "Work Related/프로젝트/스마트계류장/TODO.md", "Programing/Windows/PowerShell 에서 작업 스케쥴 생성.md", "Programing/Windows/WIndows OpenSSH server 설치.md", @@ -237,7 +209,6 @@ "Programing/Oracle/일자 일괄 생성.md", "Programing/Tibero/Lock 조회.md", "0.바로가기/DB_Lock확인.md", - "Work Related/1.업무메모/2022.05/2022-05-19.md", "zzz.resources/Pasted image 20240404100942.png", "zzz.resources/Pasted image 20240404100125.png", "zzz.resources/Pasted image 20240404095641.png", @@ -259,7 +230,6 @@ "Programing/Linux/Unix", "Work Related/프로젝트/스마트계류장/Java PGM/Radar/desktop.ini", "Work Related/프로젝트/스마트계류장/Java PGM/desktop.ini", - "무제 파일 2.canvas", - "무제 파일 1.canvas" + "무제 파일 2.canvas" ] } \ No newline at end of file diff --git a/Recognition/Programing/ELK/ElasticSearch 인덱스 통계.md b/Recognition/Programing/ELK/ElasticSearch 인덱스 통계.md new file mode 100644 index 0000000..8e78049 --- /dev/null +++ b/Recognition/Programing/ELK/ElasticSearch 인덱스 통계.md @@ -0,0 +1,274 @@ +#EKL + +- 평균 저장 건수(일,시,분,초) +```json + +GET replay_log_9_ptz-2024-05-05/_search +{ + "size": 0, + "aggs": { + "term": { + "terms": { + "field": "header.msg_type.keyword" + }, + "aggs": { + "stats": { + "stats": { + "field": "@timestamp" + } + }, + "avg_day": { + "bucket_script": { + "buckets_path": { + "max" : "stats.max", + "min" : "stats.min", + "count" : "stats.count" + }, + "script": "params.count/((params.max - params.min)/(1000 * 60 *60 * 24 ))" + } + }, + "avg_hour": { + "bucket_script": { + "buckets_path": { + "max" : "stats.max", + "min" : "stats.min", + "count" : "stats.count" + }, + "script": "params.count/((params.max - params.min)/ (1000 * 60 *60 ))" + } + }, + "avg_min": { + "bucket_script": { + "buckets_path": { + "max" : "stats.max", + "min" : "stats.min", + "count" : "stats.count" + }, + "script": "params.count/((params.max - params.min)/(1000 * 60))" + } + }, + "avg_seq": { + "bucket_script": { + "buckets_path": { + "max" : "stats.max", + "min" : "stats.min", + "count" : "stats.count" + }, + "script": "params.count/((params.max - params.min)/1000)" + } + } + + } + } + } +} + +``` + +- 누적 메시지 건수, 메시지 사이즈 +```json + +GET replay_log_1_*/_stats?filter_path=_all.total.store.size_in_bytes,_all.total.docs.count + +``` + +- 인덱스별 최종 수신시각, 건수 +```json + +GET _cat/count/replay_log_4_*?v + +``` + +- 인덱스별 건수,용량 +```json + +GET _cat/indices?v + +``` + +- 특정 인덱스 최초/최종 수신 메시지 +```json + +POST replay_log_9*/_search +{ + "size" : 0, + "_source" : false, + "aggregations" : { + "groupby" : { + "filters" : { + "filters" : [ + { + "match_all" : { + "boost" : 1.0 + } + } + ], + "other_bucket" : false, + "other_bucket_key" : "_other_" + }, + "aggregations" : { + "last_msg" : { + "top_hits" : { + "from" : 0, + "size" : 1, + "version" : false, + "seq_no_primary_term" : false, + "explain" : false, + "docvalue_fields" : [ + { + "field" : "header.recv_time.keyword" + } + ], + "sort" : [ + { + "header.recv_time.keyword" : { + "order" : "desc", + "missing" : "_last", + "unmapped_type" : "text" + } + } + ] + } + }, + "first_msg" : { + "top_hits" : { + "from" : 0, + "size" : 1, + "version" : false, + "seq_no_primary_term" : false, + "explain" : false, + "docvalue_fields" : [ + { + "field" : "header.recv_time.keyword" + } + ], + "sort" : [ + { + "header.recv_time.keyword" : { + "order" : "asc", + "missing" : "_last", + "unmapped_type" : "text" + } + } + ] + } + } + } + } + } +} + + +``` + +- 시간당 평균 건수 +```json + +POST replay_log_10*/_search +{ + "size" : 0, + "query" : { + "script" : { + "script" : { + "source" : "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalSqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", + "lang" : "painless", + "params" : { + "v0" : "header.recv_time.keyword", + "v1" : 1, + "v2" : 8, + "v3" : "20240502" + } + }, + "boost" : 1.0 + } + }, + "_source" : false, + "aggregations" : { + "groupby" : { + "composite" : { + "size" : 1000, + "sources" : [ + { + "46602450" : { + "terms" : { + "script" : { + "source" : "InternalSqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)", + "lang" : "painless", + "params" : { + "v0" : "header.recv_time.keyword", + "v1" : 9, + "v2" : 2 + } + }, + "missing_bucket" : true, + "value_type" : "string", + "order" : "asc" + } + } + } + ] + } + } + } +} + + +``` + +- 기타 +```json + +// cluster 전체 통계 조회 +GET /_cluster/stats + +// 인덱스 통계 조회 +_stats + +// 사용가능한 _cat API 목록 출려 +_cat +?v : 결과 header 출력 +?help : 각 컬럼(header) 상세 정보 출력 +?h={column name},{column name} : 선택 컬럼만 출력 +------------------------------------------------- +=^.^= +/_cat/allocation +/_cat/shards +/_cat/shards/{index} +/_cat/master +/_cat/nodes +/_cat/tasks +/_cat/indices +/_cat/indices/{index} +/_cat/segments +/_cat/segments/{index} +/_cat/count +/_cat/count/{index} +/_cat/recovery +/_cat/recovery/{index} +/_cat/health +/_cat/pending_tasks +/_cat/aliases +/_cat/aliases/{alias} +/_cat/thread_pool +/_cat/thread_pool/{thread_pools} +/_cat/plugins +/_cat/fielddata +/_cat/fielddata/{fields} +/_cat/nodeattrs +/_cat/repositories +/_cat/snapshots/{repository} +/_cat/templates +/_cat/ml/anomaly_detectors +/_cat/ml/anomaly_detectors/{job_id} +/_cat/ml/trained_models +/_cat/ml/trained_models/{model_id} +/_cat/ml/datafeeds +/_cat/ml/datafeeds/{datafeed_id} +/_cat/ml/data_frame/analytics +/_cat/ml/data_frame/analytics/{id} +/_cat/transforms +/_cat/transforms/{transform_id} +------------------------------------------------- + + +``` \ No newline at end of file diff --git a/Recognition/Programing/ELK/ElasticSearch 쿼리.md b/Recognition/Programing/ELK/ElasticSearch 쿼리.md index bea227e..0218dce 100644 --- a/Recognition/Programing/ELK/ElasticSearch 쿼리.md +++ b/Recognition/Programing/ELK/ElasticSearch 쿼리.md @@ -168,6 +168,28 @@ GET _sql/translate "query": "SELECT header.recv_time, message FROM \"*-fusion\" WHERE header.recv_time >= '20240111120000.000' AND header.recv_time <= '20240111163000.000' and header.msg_type like '3' order by header.recv_time" } +GET _sql/translate +{ + "query" : "select max(header.recv_time) as max_time, min(header.recv_time) as min_time from \"replay_log_10_route-2024-03-12\" where header.msg_type in ('3', '7')" +} + +GET _sql/translate +{ + "query" : "select max(header.recv_time) as max_time, min(header.recv_time) as min_time from \"replay_log_10_route-*\" " +} + + +GET _sql/translate +{ + "query" : "select SUBSTRING(header.recv_time, 9,2) as hour, count(*) as cnt from \"replay_log_10_route-2024-03-12\" where SUBSTRING(header.recv_time, 1,8) = '20240502' group by SUBSTRING(header.recv_time, 9,2) " +} + + +GET _sql/translate +{ + "query" : "select count(SUBSTRING(header.recv_time.keyword, 9,2)) as cnt from \"replay_log_10_route-2024-03-12\" where SUBSTRING(header.recv_time.keyword, 1,8) = '20240502' group by header.msg_type.keyword " +} + ``` ## 조회 ```json diff --git a/Recognition/Programing/ELK/무제 파일.md b/Recognition/Programing/ELK/무제 파일.md deleted file mode 100644 index e69de29..0000000 diff --git a/Recognition/Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md b/Recognition/Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md index df5227f..d836f5d 100644 --- a/Recognition/Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md +++ b/Recognition/Work Related/프로젝트/스마트계류장/0.관련정보/미들웨어 API.md @@ -1,3 +1,3 @@ http://10.200.31.42:8090/getSQL -http://10.200.31.42:8090/Select/sacp/PN_V_DATA_BLOCK_NEW \ No newline at end of file +http://10.200.31.42:8090/Select/sacp/GetM_NOTAM \ No newline at end of file diff --git a/Recognition/무제 파일 1.canvas b/Recognition/무제 파일 1.canvas new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/Recognition/무제 파일 1.canvas @@ -0,0 +1 @@ +{} \ No newline at end of file