From 20e498e8856de828eef3398a331a75df9657f507 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 23 Sep 2026 17:01:25 +0500 Subject: [PATCH] Add alert manager and telegram notifications --- group_vars/all/vault.yml | 50 ++++++++++-------- .../tasks/main.yml | 2 + templates/alertmanager.yml.j2 | 20 +++++++ templates/alerts.rules.yml.j2 | 30 +++++++++++ templates/docker-compose-https.yml.j2 | 12 ++++- templates/docker-compose.yml.j2 | 13 ++++- .../docker-compose.yml.j2:Zone.Identifier | Bin 25 -> 0 bytes .../grafana-datasource.yml.j2:Zone.Identifier | Bin 25 -> 0 bytes templates/nginx-https.conf.j2 | 1 + .../node-exporter-web.yml.j2:Zone.Identifier | Bin 25 -> 0 bytes .../node_exporter.service.j2:Zone.Identifier | Bin 25 -> 0 bytes .../prometheus-web.yml.j2:Zone.Identifier | Bin 25 -> 0 bytes templates/prometheus.yml.j2 | 6 +++ templates/prometheus.yml.j2:Zone.Identifier | Bin 25 -> 0 bytes 14 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 templates/alertmanager.yml.j2 create mode 100644 templates/alerts.rules.yml.j2 delete mode 100644 templates/docker-compose.yml.j2:Zone.Identifier delete mode 100644 templates/grafana-datasource.yml.j2:Zone.Identifier delete mode 100644 templates/node-exporter-web.yml.j2:Zone.Identifier delete mode 100644 templates/node_exporter.service.j2:Zone.Identifier delete mode 100644 templates/prometheus-web.yml.j2:Zone.Identifier delete mode 100644 templates/prometheus.yml.j2:Zone.Identifier diff --git a/group_vars/all/vault.yml b/group_vars/all/vault.yml index 07bba5e..064551a 100644 --- a/group_vars/all/vault.yml +++ b/group_vars/all/vault.yml @@ -1,23 +1,29 @@ $ANSIBLE_VAULT;1.1;AES256 -38393662663161356233646332336136363365633235656430363164336363363337633631626238 -6436646231663738626138393730316535316635356533320a636164373361613630323932616436 -31316135393966316133373333303630326435633361336634393262396665386564346264653335 -3636353436366430380a656566646364356330333536306362353631616264646165313465643763 -63396166613632376630363935616436653736663637313463666362323439323133313563613837 -33353636326466626163663736316134666464373264663539333837336463626362323738636466 -34396330333931323038316464363138386161633532613032366563636235373735343737633334 -63363865313937626637626432613065353763643334373533333365643634623539613362383535 -61313238363662626433333534613566326466303364346639653266626131356637643739336436 -33396135363037613030643237376334613433306464366532353030373532303665363463323839 -66383234626563653633373266326133333434343631656139383265626537636239343163613865 -38383131306462616334313261336534396166616164393736356565396562356361623765666633 -35386263626566346137343434303563626531626336363966643436383263363562626632613138 -31303234376339376339323561343464636666326361356236343231306138333766363832353933 -35613035643437613235343661636133636462383236326361346137333662633663313662386535 -37383933623436396663396365303831323061353937376264613431666264626161643938386237 -37356166373736323761656261323934643739663265636630663734363438386138313332386130 -39376436363936613435323032356235613563633333336332323838643331623966643130383566 -65323839653633373331373139636465663238633861623431306362343662333730656430633266 -63323131343735326264366337393432303363623362316133363335386434653932623063323037 -63373334663735373962626337346365326537663363633739316133643930616538323766663563 -3963316330643230666636303563626333373830326561636631 +63636435633636653931346361356234643336663036336236623832643231343332366561393636 +3331633566376161363130316266343632633938313032360a613363646634333237623139633063 +36663632303261383832626530373665336466386265303033643437353435633035343936313566 +3338626365653237330a303161313161613731396266383862633130663665376139363066663436 +39653264363830393436613466633532393865656264623336386539303431343864663530623839 +34616439653830303764313437313435663730346631343432323437313538303963373935336434 +38646232643334303065346533346166363732623634316634363861663366346165333863616662 +63353265643635373062386364353832376132313661613366393762613663626536663933393632 +64623864656130366330346233353732313231666131376565626566316131303230353933336633 +31663333333765363335343330373162333638303134366463383763356264323632613563666430 +34616462663662643466383866346133373939386361343661346537326338363233653130393130 +61633865656239396434616535373362313766373636323961643766343965343136636531653264 +37663163383339613565373433326365633331636365616665386361353835636265343035666463 +61316635666132326130356161623237353034633561393634396663383936363439313863343737 +62666166326564666365393830363830333865333230376339366138356261376166313230363566 +62373930316632613135346531646336373232306137656262343636613433616366383962613864 +30623131323633623439393765346239363531633734636561613231393661383938373739373033 +63653238393832656333343139636239366164386133623630643333626464343563636361666337 +30313934633561633434303961343561343033316362623239383634306139373830366566393661 +30356331663134613462366436383036366432336466386166303362333531636262343534626666 +61613331633136646664373538323363363461643338376466366436313166376634633566356365 +63316366386536336662383961373338626132313363626333663531326166656637633661343032 +65386534636266386634633263613934636565313364323834326237376363353435386237363637 +61663233323637396132633832636265303864393037376536353437643265353034306636343537 +61363964613138616464336165623962363033356261373937656364396131366433376532383436 +65613262373733623164303832653362656235623664353561326461376333616262653536306461 +63376336396533656664323065663432306137363432363563356237343436653931633731646137 +34313936366439623438 diff --git a/roles/deploy_prometheus_and_grafana/tasks/main.yml b/roles/deploy_prometheus_and_grafana/tasks/main.yml index 23f40ae..35a973e 100644 --- a/roles/deploy_prometheus_and_grafana/tasks/main.yml +++ b/roles/deploy_prometheus_and_grafana/tasks/main.yml @@ -30,6 +30,8 @@ - { src: "grafana-datasource.yml.j2", dest: "grafana_provisioning/datasource.yml" } - { src: "docker-compose.yml.j2", dest: "docker-compose.yml" } - { src: "docker-compose-https.yml.j2", dest: "docker-compose-https.yml" } + - { src: "alertmanager.yml.j2", dest: "alertmanager.yml" } + - { src: "alerts.rules.yml.j2", dest: "alerts.rules.yml" } notify: Restart docker-compose - name: Скопировать конфиг nginx diff --git a/templates/alertmanager.yml.j2 b/templates/alertmanager.yml.j2 new file mode 100644 index 0000000..f4a3ff1 --- /dev/null +++ b/templates/alertmanager.yml.j2 @@ -0,0 +1,20 @@ +route: + receiver: 'telegram' + group_wait: 30s + repeat_interval: 1h +receivers: + - name: 'telegram' + telegram_configs: + - bot_token: "{{ telegram_bot_token }}" + chat_id: {{ telegram_chat_id }} + message: >- + {% raw %} + {{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} + *{{ .GroupLabels.alertname }}* + + {{ range .Alerts }} + *Сервер:* `{{ .Labels.instance }}` + *Статус:* {{ .Status }} + *Описание:* {{ .Annotations.description }} + {{ end }} + {% endraw %} diff --git a/templates/alerts.rules.yml.j2 b/templates/alerts.rules.yml.j2 new file mode 100644 index 0000000..7822643 --- /dev/null +++ b/templates/alerts.rules.yml.j2 @@ -0,0 +1,30 @@ +groups: + - name: main + rules: + + # Алерт 1: Сервер недоступен + - alert: InstanceDown + expr: up == 0 + for: 1m + annotations: + description: "{% raw %}Сервер {{ $labels.instance }} недоступен{% endraw %}" + + # Алерт 2: Высокая загрузка CPU (любой сервер) + - alert: HighCpuLoad + expr: (1 - avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[1m]))) * 100 > 80 + for: 1m + annotations: + description: >- + {% raw %}Сервер {{ $labels.instance }}: загрузка CPU {{ printf "%.1f" $value }}% (порог 80%){% endraw %} + + # Алерт 3: Высокий трафик на general_server (сумма вход + выход > 10 МБ/с) + - alert: HighNetworkTraffic + expr: >- + sum by(instance) ( + rate(node_network_receive_bytes_total{nodename="general_server", device!="lo"}[1m]) + + rate(node_network_transmit_bytes_total{nodename="general_server", device!="lo"}[1m]) + ) / 1024 / 1024 > 10 + for: 1m + annotations: + description: >- + {% raw %}general_server: суммарный трафик {{ printf "%.1f" $value }} МБ/с превысил порог 10 МБ/с{% endraw %} \ No newline at end of file diff --git a/templates/docker-compose-https.yml.j2 b/templates/docker-compose-https.yml.j2 index 63aa3ea..08bb9fc 100644 --- a/templates/docker-compose-https.yml.j2 +++ b/templates/docker-compose-https.yml.j2 @@ -7,6 +7,7 @@ services: volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./prometheus-web.yml:/etc/prometheus/prometheus-web.yml:ro + - ./alerts.rules.yml:/etc/prometheus/alerts.rules.yml:ro - prom_data:/prometheus command: @@ -22,7 +23,7 @@ services: image: grafana/grafana:latest container_name: grafana ports: - - "3001:3000" + - "127.0.0.1:3001:3000" volumes: - ./grafana_provisioning/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro - grafana_data:/var/lib/grafana @@ -37,6 +38,15 @@ services: depends_on: - prometheus + alertmanager: + image: prom/alertmanager:latest + container_name: alertmanager + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml + ports: + - "127.0.0.1:9093:9093" + restart: unless-stopped + volumes: prom_data: grafana_data: diff --git a/templates/docker-compose.yml.j2 b/templates/docker-compose.yml.j2 index ccc5ef9..8cdffb8 100644 --- a/templates/docker-compose.yml.j2 +++ b/templates/docker-compose.yml.j2 @@ -7,6 +7,7 @@ services: volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./prometheus-web.yml:/etc/prometheus/prometheus-web.yml:ro + - ./alerts.rules.yml:/etc/prometheus/alerts.rules.yml:ro - prom_data:/prometheus command: @@ -22,7 +23,7 @@ services: image: grafana/grafana:latest container_name: grafana ports: - - "127.0.0.1:3001:3000" + - "3001:3000" volumes: - ./grafana_provisioning/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro - grafana_data:/var/lib/grafana @@ -37,6 +38,16 @@ services: depends_on: - prometheus + alertmanager: + image: prom/alertmanager:latest + container_name: alertmanager + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml + ports: + - "127.0.0.1:9093:9093" + restart: unless-stopped + + volumes: prom_data: grafana_data: diff --git a/templates/docker-compose.yml.j2:Zone.Identifier b/templates/docker-compose.yml.j2:Zone.Identifier deleted file mode 100644 index d6c1ec682968c796b9f5e9e080cc6f674b57c766..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25 dcma!!%Fjy;DN4*MPD?F{<>dl#JyUFr831@K2xdl#JyUFr831@K2xdl#JyUFr831@K2xdl#JyUFr831@K2xdl#JyUFr831@K2xdl#JyUFr831@K2x