{ "schema_version": 1, "generated_at": "2026-06-08T15:33:44+03:00", "weeks": [ { "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true }, { "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "families": { "delivery": true, "tests": false, "alerts": true }, "has_any_report": true }, { "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "families": { "delivery": true, "tests": true, "alerts": true }, "has_any_report": true } ], "items": [ { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 9, "last_seen": "2026-03-20 11:11", "last_seen_ts": 1773997882.351599, "services": "docgen2-api (6), subscriptions-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773997882351599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773997882351599", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 27, "last_seen": "2026-03-19 07:46", "last_seen_ts": 1773899212.809979, "services": "admission-end-session (27), update-recurenta (13)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773899212809979", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773899212809979", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-03-18 14:53", "last_seen_ts": 1773838409.742669, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773838409742669", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773838409742669", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-03-19 12:27", "last_seen_ts": 1773916039.689679, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773916039689679", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1773916039689679", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 17, "last_seen": "2026-03-20 21:47", "last_seen_ts": 1774036060.731799, "services": "uni-api-svc-4000 (12), accommodations-api-svc-4100 (5)", "signals": "General investigation, Observability storage", "latest_note": "SQL Integrity Constraint Violation: (conn=3570102) Cannot add or update a child row: a foreign key constraint fails (`ums_uni_catalog`.`dis… | Andrei Alexandru", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774036060731799", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774036060731799?thread_ts=1774036060.731799&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-03-20 16:35", "last_seen_ts": 1774017308.169829, "services": "ai-api-svc-3900 (4), web-80 (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774017308169829", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774017308169829", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 2, "last_seen": "2026-03-21 00:00", "last_seen_ts": 1774044000.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-master-memory-low, adservio-rds-mysql-master-write-latency-high", "latest_note": "adservio-rds-mysql-master-memory-low, adservio-rds-mysql-master-write-latency-high", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-memory-low", "item_label": "adservio-rds-mysql-master-memory-low", "severity": "critical", "status": "Still alarming", "count": 63, "last_seen": "2026-03-18 07:09", "last_seen_ts": 1773810594.0, "services": "", "signals": "ALARM 32, OK 31, Flips 62", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-14T07:00:00+02:00", "period_end": "2026-03-21T07:00:00+02:00", "week_key": "2026-03-14T07:00__2026-03-21T07:00", "week_label": "W11", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-write-latency-high", "item_label": "adservio-rds-mysql-master-write-latency-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-03-16 23:13", "last_seen_ts": 1773695602.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.html", "source_json": "production_reliability_dashboard_2026-03-14_0700_2026-03-21_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 5, "last_seen": "2026-03-27 13:05", "last_seen_ts": 1774609552.342729, "services": "docgen2-api (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774609552342729", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774609552342729", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 32, "last_seen": "2026-03-28 05:12", "last_seen_ts": 1774667542.570389, "services": "accommodations-sync-users-29572740 (21), download-album-29576545 (6), accommodations-sync-users-29571300 (5)", "signals": "General investigation", "latest_note": "production.ERROR: Allowed memory size of 134217728 bytes exhausted (tried to allocate 200704 bytes) {\"exception\":\"[object] (Symfony\\\\Compon… | Ionut Ciolan a dat out of memory. E setat cumva in php.ini max memory la 128mb? Ai cum sa o cresti sa nu mai crape? | mai bine ar fi sa faci un pic refactor la cod. Pare ca inc…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774667542570389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774342311275009?thread_ts=1774342311.275009&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "KubeNodeEviction", "item_label": "KubeNodeEviction", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-03-26 08:38", "last_seen_ts": 1774507123.429909, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774507123429909", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774507123429909", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-03-27 11:29", "last_seen_ts": 1774603799.695859, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603799695859", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603799695859", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-03-27 11:29", "last_seen_ts": 1774603789.674389, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603789674389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774603789674389", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 2, "last_seen": "2026-03-27 08:23", "last_seen_ts": 1774592623.169059, "services": "uni-api-svc-4000 (1), web-80 (1)", "signals": "General investigation", "latest_note": "Andrei Alexandru pare ca e o buba cu foreign keys pe tuiasi | Service method DisciplineServiceImpl.patchDisciplineByIdAndAcademicPlan failed after 12ms: could not execute statement [(conn=4451547) Cann… | ok, urmaresc si rezolv, am mai gasit ieri.o astfel de problema, o urmaresc si pe asta", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774592623169059", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774429093165839?thread_ts=1774429093.165839&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen this week", "count": 16, "last_seen": "2026-03-27 17:21", "last_seen_ts": 1774624868.180819, "services": "ai-api-svc-3900 (11), web-80 (5), subscriptions-api-svc-3400 (4), rooms-api-svc-3700 (1)", "signals": "General investigation", "latest_note": "era un query la catalog care mergea greu", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774624868180819", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1774516778168249?thread_ts=1774516778.168249&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-03-27 08:41", "last_seen_ts": 1774593708.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-memory-low", "item_label": "adservio-rds-mysql-catalog-memory-low", "severity": "critical", "status": "Still alarming", "count": 2, "last_seen": "2026-03-27 08:41", "last_seen_ts": 1774593678.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-swap-high", "item_label": "adservio-rds-mysql-catalog-swap-high", "severity": "critical", "status": "Still alarming", "count": 2, "last_seen": "2026-03-27 12:14", "last_seen_ts": 1774606482.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-memory-low", "item_label": "adservio-rds-mysql-catalog2-memory-low", "severity": "critical", "status": "Still alarming", "count": 43, "last_seen": "2026-03-25 10:11", "last_seen_ts": 1774426269.0, "services": "", "signals": "ALARM 22, OK 21, Flips 42", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-21T07:00:00+02:00", "period_end": "2026-03-28T07:00:00+02:00", "week_key": "2026-03-21T07:00__2026-03-28T07:00", "week_label": "W12", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-swap-high", "item_label": "adservio-rds-mysql-catalog2-swap-high", "severity": "critical", "status": "Still alarming", "count": 1, "last_seen": "2026-03-27 13:43", "last_seen_ts": 1774611823.0, "services": "", "signals": "ALARM 1, OK 0, Flips 0", "latest_note": "ALARM", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.html", "source_json": "production_reliability_dashboard_2026-03-21_0700_2026-03-28_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 10, "last_seen": "2026-04-04 04:19", "last_seen_ts": 1775265587.844989, "services": "metrics-server (10)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775265587844989", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775265587844989", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-04-03 10:18", "last_seen_ts": 1775200707.893329, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775200707893329", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775200707893329", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 26, "last_seen": "2026-04-02 15:44", "last_seen_ts": 1775133891.047499, "services": "accommodations-events (25), metrics-server (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775133891047499", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775133891047499", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 7, "last_seen": "2026-04-03 13:55", "last_seen_ts": 1775213722.350009, "services": "docgen2-api (4), grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775213722350009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775213722350009", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 54, "last_seen": "2026-04-04 04:36", "last_seen_ts": 1775266612.837159, "services": "accommodations-sync-users (31), download-album (13), update-recurenta (10), minicrm-sync (8)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775266612837159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775266612837159", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-02 15:05", "last_seen_ts": 1775131552.349969, "services": "update-recurenta (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775131552349969", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775131552349969", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-04-02 14:29", "last_seen_ts": 1775129361.085209, "services": "accommodations-events (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775129361085209", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775129361085209", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-01 10:54", "last_seen_ts": 1775030092.344839, "services": "core-grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775030092344839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775030092344839", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 1, "last_seen": "2026-04-03 09:24", "last_seen_ts": 1775197483.233569, "services": "docgen2-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775197483233569", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775197483233569", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-03 19:02", "last_seen_ts": 1775232158.182759, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775232158182759", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775232158182759", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 2, "last_seen": "2026-04-04 00:00", "last_seen_ts": 1775250000.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-04-02 10:22", "last_seen_ts": 1775114523.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-03-28T07:00:00+02:00", "period_end": "2026-04-04T07:00:00+03:00", "week_key": "2026-03-28T07:00__2026-04-04T07:00", "week_label": "W13", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-03 13:50", "last_seen_ts": 1775213434.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.html", "source_json": "production_reliability_dashboard_2026-03-28_0700_2026-04-04_0700.json" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-04-06 17:34", "last_seen_ts": 1775486089.032009, "services": "metrics-server (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775486089032009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775486089032009", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-04-08 02:53", "last_seen_ts": 1775606002.531429, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775606002531429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775606002531429", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 44, "last_seen": "2026-04-11 03:30", "last_seen_ts": 1775867452.435479, "services": "update-recurenta (44)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775867452435479", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775867452435479", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobNotCompleted", "item_label": "KubeJobNotCompleted", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-04-09 03:20", "last_seen_ts": 1775694052.569049, "services": "update-recurenta (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775694052569049", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775694052569049", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 2, "last_seen": "2026-04-06 00:44", "last_seen_ts": 1775425453.269459, "services": "core-grafana-80 (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775425453269459", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775425453269459", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-04T07:00:00+03:00", "period_end": "2026-04-11T07:00:00+03:00", "week_key": "2026-04-04T07:00__2026-04-11T07:00", "week_label": "W14", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-04-10 17:25", "last_seen_ts": 1775831138.187949, "services": "ai-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775831138187949", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1775831138187949", "source_href": "reports/production_reliability_dashboard_2026-04-04_0700_2026-04-11_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Recent (72h)", "count": 1, "last_seen": "2026-04-16 07:29", "last_seen_ts": 1776313798.456709, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776313798456709", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776313798456709", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-17 10:18", "last_seen_ts": 1776410307.906639, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410307906639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410307906639", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen today", "count": 2, "last_seen": "2026-04-17 13:28", "last_seen_ts": 1776421732.339979, "services": "accommodations-api (1), library-api (1), rooms-api (1), web (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421732339979", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421732339979", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-04-17 09:51", "last_seen_ts": 1776408682.407399, "services": "docgen2-api (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776408682407399", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776408682407399", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 44, "last_seen": "2026-04-18 03:33", "last_seen_ts": 1776472432.595369, "services": "update-recurenta (39), colecteaza-sms-note-abs (6), download-album (5), rezumat (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776472432595369", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776472432595369", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-04-17 10:19", "last_seen_ts": 1776410391.066099, "services": "accommodations-events (2), library-api (2), rooms-api (2), accommodations-api (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410391066099", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776410391066099", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Seen today", "count": 3, "last_seen": "2026-04-17 14:01", "last_seen_ts": 1776423682.323779, "services": "web (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776423682323779", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776423682323779", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen today", "count": 1, "last_seen": "2026-04-17 13:24", "last_seen_ts": 1776421497.821429, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421497821429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776421497821429", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-04-17 18:22", "last_seen_ts": 1776439328.183589, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776439328183589", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776439328183589", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-11T07:00:00+03:00", "period_end": "2026-04-18T07:00:00+03:00", "week_key": "2026-04-11T07:00__2026-04-18T07:00", "week_label": "W15", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-15 12:26", "last_seen_ts": 1776245205.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-11_0700_2026-04-18_0700.html", "source_json": "" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeContainerWaiting", "item_label": "KubeContainerWaiting", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-04-20 12:38", "last_seen_ts": 1776677902.384839, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776677902384839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776677902384839", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 24, "last_seen": "2026-04-24 12:29", "last_seen_ts": 1777022967.925309, "services": "grafana (24)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777022967925309", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777022967925309", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-04-20 11:48", "last_seen_ts": 1776674902.445639, "services": "core-getresponse-events-worker (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776674902445639", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776674902445639", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentRolloutStuck", "item_label": "KubeDeploymentRolloutStuck", "severity": "warning", "status": "Likely noise / resolved", "count": 2, "last_seen": "2026-04-20 17:27", "last_seen_ts": 1776695272.342619, "services": "forms-api (2)", "signals": "Alert tuning / noise", "latest_note": "Investigation summary for `forms-api` | | *What happened* | - The `forms-api` rollout stalled and triggered `KubeDeploymentRolloutStuck`. |… | , @U09JYAWCGLB @U633D9JBW I made this skill that can run automatically as soon as an alert appears in Slack, so it ca… | we can improve it further based o…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695272342619", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695272342619?thread_ts=1776695272.342619&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-04-22 09:47", "last_seen_ts": 1776840472.369839, "services": "docgen2-api (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776840472369839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776840472369839", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 27, "last_seen": "2026-04-22 13:48", "last_seen_ts": 1776854934.134389, "services": "update-recurenta (27), colecteaza-sms-note-abs (14), download-album (14), rezumat (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776854934134389", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776854934134389", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-04-20 17:24", "last_seen_ts": 1776695062.359829, "services": "forms-api (2), update-recurenta (1)", "signals": "General investigation", "latest_note": "Warning Unhealthy 80s (x19 over 3m22s) kubelet spec.containers{forms-api-container}: Readiness probe failed: Get \"\": dial tcp 10.66.111.240… | one of the pod is healthy to keep the service alive . | Thats right", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776695062359829", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776692782380059?thread_ts=1776692782.380059&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 7, "last_seen": "2026-04-21 12:41", "last_seen_ts": 1776764502.679159, "services": "core-grafana-80 (3), admission-api (3), uni-api (1)", "signals": "General investigation", "latest_note": "@U09JYAWCGLB we know about this, discussing in daily with the devs | Hi Raul Popovici sure thanks", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776764502679159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1776753193171579?thread_ts=1776753193.171579&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 7, "last_seen": "2026-04-24 17:35", "last_seen_ts": 1777041338.182049, "services": "ai-api (4), social-api (3)", "signals": "Resource limits, General investigation", "latest_note": "some POSTs to ai summaries seem to take 5s or more. But we don't have traces in ai-api pod to see if it is normal (summaries could indeed t…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777041338182049", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777041338182049?thread_ts=1777041338.182049&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-20 10:14", "last_seen_ts": 1776669285.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog2-storage-low", "item_label": "adservio-rds-mysql-catalog2-storage-low", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-04-20 13:52", "last_seen_ts": 1776682338.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-18T07:00:00+03:00", "period_end": "2026-04-25T07:00:00+03:00", "week_key": "2026-04-18T07:00__2026-04-25T07:00", "week_label": "W16", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-root-account-usage", "item_label": "adservio-root-account-usage", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-04-23 15:32", "last_seen_ts": 1776947574.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.html", "source_json": "production_reliability_dashboard_2026-04-18_0700_2026-04-25_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 43, "last_seen": "2026-05-04 18:19", "last_seen_ts": 1777907988.501169, "services": "metrics-server (43)", "signals": "General investigation", "latest_note": "False alarm metrics-server pod is healthy (1/1 Running, 0 restarts) and kubectl top nodes works. Alert was a brief flap during a node rotat…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777273195154609?thread_ts=1777273195.154609&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAPIErrorBudgetBurn", "item_label": "KubeAPIErrorBudgetBurn", "severity": "critical", "status": "Likely noise / resolved", "count": 3, "last_seen": "2026-04-27 12:48", "last_seen_ts": 1777283323.254389, "services": "grafana (4)", "signals": "General investigation, Alert tuning / noise", "latest_note": "Nothing is down apiserver healthy (/readyz ok), all pods Running, HPAs scaled back down (web 9, docgen2 4). Alert is the rolling 1h SLO win… | you can silence it in alertmanger | Okay i will try a bit later thanks for the info", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777293458219179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777283323254389?thread_ts=1777283323.254389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeClientErrors", "item_label": "KubeClientErrors", "severity": "warning", "status": "No recent signal", "count": 1, "last_seen": "2026-04-27 12:09", "last_seen_ts": 1777280977.725749, "services": "grafana (1)", "signals": "General investigation", "latest_note": "aiserver is healthy (/readyz ok). The 2% error rate is from admission webhook timeouts during the HPA scaling burst (web 11 replicas, docge…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280977725749", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280977725749?thread_ts=1777280977.725749&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 20, "last_seen": "2026-05-04 13:12", "last_seen_ts": 1777889577.894069, "services": "grafana (20)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Recent (72h)", "count": 13, "last_seen": "2026-05-03 07:09", "last_seen_ts": 1777781361.612109, "services": "rooms-api (13), core-scheduled-events-worker (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-04 12:17", "last_seen_ts": 1777886272.344179, "services": "docgen2-api (5)", "signals": "Resource limits", "latest_note": "HPA already scaled back down (now 4/6). Pods are barely using CPU, so the scale-up was driven by a KEDA cron trigger, not real load.", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777280212372279?thread_ts=1777280212.372279&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen today", "count": 23, "last_seen": "2026-05-04 19:21", "last_seen_ts": 1777911712.019249, "services": "minicrm-sync (23), colecteaza-sms-note-abs (21), accommodations-sync-users (19)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-04-29 09:26", "last_seen_ts": 1777443987.935159, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777443987935159", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777443987935159", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Recent (72h)", "count": 14, "last_seen": "2026-05-03 07:14", "last_seen_ts": 1777781691.594009, "services": "rooms-api (13), core-scheduled-events-worker (3)", "signals": "General investigation", "latest_note": "danny Valentin Pal something is breaking on the Tuiasi cluster which i am not able to connect | | i would need a kubeconfig for tuiasi and…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777605291164119?thread_ts=1777605291.164119&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-04-30 12:30", "last_seen_ts": 1777541449.731469, "services": "grafana (1)", "signals": "Resource limits", "latest_note": "Node 10.66.121.122 is at 79% CPU / 50% memory, no pressure conditions load is 2.27 because 5 of the web pods landed on the same node plus s… | Will clear when traffic dies down or HPA rebalances | Stabilized load on 10.66.121.122 dropped from 2.27 to 1.68 per core (15m avg), CPU from 79% to 55%. HPA scaled down + web…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777541449731469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777541449731469?thread_ts=1777541449.731469&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 26, "last_seen": "2026-05-04 15:06", "last_seen_ts": 1777896400.713119, "services": "accommodations-api (10), web-80 (8), uni-api (7), core-grafana-80 (1)", "signals": "General investigation, Observability storage", "latest_note": "uni-api is throwing 500s because the same academic-structure save is being submitted twice second insert hits a unique-key constraint (stud…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777896400713119", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777883293167389?thread_ts=1777883293.167389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 18, "last_seen": "2026-05-04 12:39", "last_seen_ts": 1777887548.184509, "services": "social-api (10), ai-api (7), subscriptions-api (2), web-80 (2)", "signals": "General investigation", "latest_note": "Two outbound calls to web dominated: GET /api/v2/utilizatori/profile?cuPermisiuni=1&withRoles=1 (6.6s) and GET /api/v2/ani/scolari (6.4s) —…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509?thread_ts=1777887548.184509&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 9, "last_seen": "2026-05-05 00:00", "last_seen_ts": 1777928400.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-catalog-disk-queue-high", "latest_note": "adservio-rds-mysql-catalog-disk-queue-high", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Flapping, latest OK", "count": 14, "last_seen": "2026-04-30 10:21", "last_seen_ts": 1777533704.0, "services": "", "signals": "ALARM 7, OK 7, Flips 13", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-04-25T07:00:00+03:00", "period_end": "2026-05-02T07:00:00+03:00", "week_key": "2026-04-25T07:00__2026-05-02T07:00", "week_label": "W17", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-01 10:13", "last_seen_ts": 1777619583.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.html", "source_json": "production_reliability_dashboard_2026-04-25_0700_2026-05-02_0700.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-05-04 18:19", "last_seen_ts": 1777907988.501169, "services": "metrics-server (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777907988501169", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-04 13:12", "last_seen_ts": 1777889577.894069, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777889577894069", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-03 07:09", "last_seen_ts": 1777781361.612109, "services": "rooms-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781361612109", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-04 12:17", "last_seen_ts": 1777886272.344179, "services": "docgen2-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777886272344179", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 15, "last_seen": "2026-05-04 19:21", "last_seen_ts": 1777911712.019249, "services": "accommodations-sync-users (15), colecteaza-sms-note-abs (15), minicrm-sync (15)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777911712019249", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-03 07:14", "last_seen_ts": 1777781691.594009, "services": "rooms-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777781691594009", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 12, "last_seen": "2026-05-04 15:06", "last_seen_ts": 1777896400.713119, "services": "accommodations-api (8), uni-api (4)", "signals": "Observability storage", "latest_note": "uni-api is throwing 500s because the same academic-structure save is being submitted twice second insert hits a unique-key constraint (stud…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777896400713119", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777883293167389?thread_ts=1777883293.167389&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen this week", "count": 7, "last_seen": "2026-05-04 12:39", "last_seen_ts": 1777887548.184509, "services": "social-api (6), ai-api (1)", "signals": "General investigation", "latest_note": "Two outbound calls to web dominated: GET /api/v2/utilizatori/profile?cuPermisiuni=1&withRoles=1 (6.6s) and GET /api/v2/ani/scolari (6.4s) —…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1777887548184509?thread_ts=1777887548.184509&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 5, "last_seen": "2026-05-09 00:00", "last_seen_ts": 1778274000.0, "services": "adservio-ro-api-v2-status", "signals": "adservio-rds-mysql-catalog-disk-queue-high, adservio-rds-mysql-catalog-storage-low", "latest_note": "adservio-rds-mysql-catalog-disk-queue-high, adservio-rds-mysql-catalog-storage-low", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-disk-queue-high", "item_label": "adservio-rds-mysql-catalog-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:30", "last_seen_ts": 1778059814.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-storage-low", "item_label": "adservio-rds-mysql-catalog-storage-low", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:26", "last_seen_ts": 1778059565.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-02T07:00:00+03:00", "period_end": "2026-05-09T07:00:00+03:00", "week_key": "2026-05-02T07:00__2026-05-09T07:00", "week_label": "W18", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-catalog-write-latency-high", "item_label": "adservio-rds-mysql-catalog-write-latency-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-05-06 12:26", "last_seen_ts": 1778059618.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.html", "source_json": "production_reliability_dashboard_2026-05-02_0700_2026-05-09_1000.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-12 10:35", "last_seen_ts": 1778571353.424169, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778571353424169", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778571353424169", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-15 15:02", "last_seen_ts": 1778846577.960959, "services": "grafana (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778846577960959", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778846577960959", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen today", "count": 14, "last_seen": "2026-05-16 05:19", "last_seen_ts": 1778897962.434189, "services": "audit-worker (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778897962434189", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778897962434189", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-12 13:48", "last_seen_ts": 1778582902.353969, "services": "docgen2-api (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778582902353969", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778582902353969", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen today", "count": 5, "last_seen": "2026-05-16 04:43", "last_seen_ts": 1778895802.601009, "services": "audit-worker (5)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778895802601009", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778895802601009", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeCPUHighUsage", "item_label": "NodeCPUHighUsage", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-05-13 09:27", "last_seen_ts": 1778653649.689149, "services": "grafana (3)", "signals": "Resource limits", "latest_note": "Node running at 98 percent CPU checking | | Web pods are the most resource dominant on the cluster | this is resolved for now , rollout restarted the web pods to replace them to the other nodes that was sitting idle to balance the load", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778653649689149", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778651549707879?thread_ts=1778651549.707879&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeHighNumberConntrackEntriesUsed", "item_label": "NodeHighNumberConntrackEntriesUsed", "severity": "warning", "status": "Seen today", "count": 23, "last_seen": "2026-05-15 12:29", "last_seen_ts": 1778837389.695469, "services": "grafana (23)", "signals": "General investigation", "latest_note": "This has been resolved", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837389695469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778750239702649?thread_ts=1778750239.702649&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-05-15 12:33", "last_seen_ts": 1778837629.699669, "services": "grafana (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837629699669", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778837629699669", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 1, "last_seen": "2026-05-13 11:55", "last_seen_ts": 1778662513.202819, "services": "uni-api (1)", "signals": "General investigation", "latest_note": "no errors now pod is healthy | | we saw this error | | Service method CareerServiceImpl.saveCareer failed after Nms: | Invalid employment t…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778662513202819", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778662513202819?thread_ts=1778662513.202819&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 25, "last_seen": "2026-05-15 19:19", "last_seen_ts": 1778861978.209259, "services": "social-api (11), ai-api (11), web-80 (6), uni-api (2)", "signals": "Release / migration issue, Scaling config", "latest_note": "12 min ago, 15+ web pod probe failures fired simultaneously (readiness /ready timeouts + php-fpm-healthcheck 10s timeouts). All 6+ web pods… | Suggesting two quick config changes on web to stop the recurring p95 latency alerts | | 1. Loosen readiness probe: failureThreshold: 4, per… | understood, good point", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778861978209259", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1778746838179519?thread_ts=1778746838.179519&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-09T07:00:00+03:00", "period_end": "2026-05-16T07:00:00+03:00", "week_key": "2026-05-09T07:00__2026-05-16T07:00", "week_label": "W19", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Recovered recently", "count": 15, "last_seen": "2026-05-15 11:49", "last_seen_ts": 1778834940.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.html", "source_json": "production_reliability_dashboard_2026-05-09_0700_2026-05-16_0700.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeCPUOvercommit", "item_label": "KubeCPUOvercommit", "severity": "warning", "status": "Seen this week", "count": 12, "last_seen": "2026-05-21 11:27", "last_seen_ts": 1779352077.869409, "services": "grafana (12)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779352077869409", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779352077869409", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeDeploymentReplicasMismatch", "item_label": "KubeDeploymentReplicasMismatch", "severity": "warning", "status": "Seen this week", "count": 35, "last_seen": "2026-05-18 15:59", "last_seen_ts": 1779109162.382199, "services": "audit-worker (35)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779109162382199", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779109162382199", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 6, "last_seen": "2026-05-21 14:16", "last_seen_ts": 1779362212.405069, "services": "grafana (3), docgen2-api (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779362212405069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779362212405069", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Recent (72h)", "count": 3, "last_seen": "2026-05-22 07:27", "last_seen_ts": 1779424072.451799, "services": "download-album (3), rezumat (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779424072451799", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779424072451799", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodCrashLooping", "item_label": "KubePodCrashLooping", "severity": "warning", "status": "Seen this week", "count": 14, "last_seen": "2026-05-18 13:53", "last_seen_ts": 1779101603.202509, "services": "audit-worker (14)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779101603202509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779101603202509", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "NodeHighNumberConntrackEntriesUsed", "item_label": "NodeHighNumberConntrackEntriesUsed", "severity": "warning", "status": "Seen this week", "count": 85, "last_seen": "2026-05-21 12:02", "last_seen_ts": 1779354169.715839, "services": "grafana (85)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779354169715839", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779354169715839", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 2, "last_seen": "2026-05-19 09:19", "last_seen_ts": 1779171589.690869, "services": "grafana (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779171589690869", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779171589690869", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen this week", "count": 3, "last_seen": "2026-05-21 16:10", "last_seen_ts": 1779369043.275199, "services": "core-grafana-80 (2), web-80 (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779369043275199", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779369043275199", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Recent (72h)", "count": 53, "last_seen": "2026-05-22 18:25", "last_seen_ts": 1779463508.176599, "services": "web-80 (37), social-api (33), ai-api (17), subscriptions-api (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779463508176599", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779463508176599", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "customer", "source_label": "Customer warning", "item_key": "Adservio Ro", "item_label": "Adservio Ro", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-20 18:13", "last_seen_ts": 1779289980.0, "services": "adservio-ro", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-16T07:00:00+03:00", "period_end": "2026-05-23T07:00:00+03:00", "week_key": "2026-05-16T07:00__2026-05-23T07:00", "week_label": "W20", "source_type": "customer", "source_label": "Customer warning", "item_key": "https://www.adservio.ro/api/v2/status", "item_label": "https://www.adservio.ro/api/v2/status", "severity": "warning", "status": "Seen this week", "count": 13, "last_seen": "2026-05-20 19:04", "last_seen_ts": 1779293040.0, "services": "adservio-ro-api-v2-status", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "https://www.adservio.ro/api/v2/status", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.html", "source_json": "production_reliability_dashboard_2026-05-16_0700_2026-05-24_2011.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Recent (72h)", "count": 12, "last_seen": "2026-05-28 14:40", "last_seen_ts": 1779968422.512619, "services": "docgen2-api (9), grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779968422512619", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779968422512619", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Recent (72h)", "count": 4, "last_seen": "2026-05-28 19:09", "last_seen_ts": 1779984598.009709, "services": "grafana (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779984598009709", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779984598009709", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "KubePodNotReady", "item_label": "KubePodNotReady", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-05-26 13:51", "last_seen_ts": 1779792682.405959, "services": "biling-efactura-worker-ff45f6d-rxxlg (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779792682405959", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779792682405959", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen today", "count": 4, "last_seen": "2026-05-29 12:26", "last_seen_ts": 1780046779.856509, "services": "grafana (4)", "signals": "Release / migration issue", "latest_note": "Worker node 10.66.133.111 (c7a.2xlarge, 8 vCPU) is CPU-saturated at 91% real usage 6 of 21 web pods landed on it (no topology spread on the… | Also durable fix is adding topologySpreadConstraints | to the web Deployment which i will look into", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780046779856509", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1779945379858529?thread_ts=1779945379.858529&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Seen today", "count": 2, "last_seen": "2026-05-29 18:04", "last_seen_ts": 1780067083.244629, "services": "core-grafana-80 (2)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067083244629", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067083244629", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 6, "last_seen": "2026-05-29 18:06", "last_seen_ts": 1780067168.183929, "services": "ai-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067168183929", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780067168183929", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-memory-low", "item_label": "adservio-rds-mysql-master-memory-low", "severity": "warning", "status": "Flapping, latest OK", "count": 13, "last_seen": "2026-05-29 12:00", "last_seen_ts": 1780045223.0, "services": "", "signals": "ALARM 6, OK 7, Flips 12", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-23T07:00:00+03:00", "period_end": "2026-05-30T07:00:00+03:00", "week_key": "2026-05-23T07:00__2026-05-30T07:00", "week_label": "W21", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 4, "last_seen": "2026-05-29 10:23", "last_seen_ts": 1780039416.0, "services": "", "signals": "ALARM 2, OK 2, Flips 3", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.html", "source_json": "production_reliability_dashboard_2026-05-23_0700_2026-05-30_0700.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "AlertmanagerFailedToSendAlerts", "item_label": "AlertmanagerFailedToSendAlerts", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-03 14:10", "last_seen_ts": 1780485048.343579, "services": "grafana (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048343579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048343579", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeAggregatedAPIDown", "item_label": "KubeAggregatedAPIDown", "severity": "warning", "status": "Seen today", "count": 27, "last_seen": "2026-06-08 00:20", "last_seen_ts": 1780867249.341069, "services": "metrics-server (27)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780867249341069", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780867249341069", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeHpaMaxedOut", "item_label": "KubeHpaMaxedOut", "severity": "warning", "status": "Seen this week", "count": 9, "last_seen": "2026-06-03 16:34", "last_seen_ts": 1780493662.476469, "services": "docgen2-api (6), grafana (2), notifications-event-manager (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780493662476469", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780493662476469", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeJobFailed", "item_label": "KubeJobFailed", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-03 14:10", "last_seen_ts": 1780485048.348319, "services": "accommodations-sync-users (1), colecteaza-sms-note-abs (1), minicrm-sync (1)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048348319", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780485048348319", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeletServerCertificateExpiration", "item_label": "KubeletServerCertificateExpiration", "severity": "warning", "status": "No recent signal", "count": 6, "last_seen": "2026-05-31 11:22", "last_seen_ts": 1780215753.737109, "services": "grafana (6)", "signals": "General investigation", "latest_note": "Cert auto-rotated over the weekend", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780215753737109", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780215753737109?thread_ts=1780215753.737109&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeMemoryOvercommit", "item_label": "KubeMemoryOvercommit", "severity": "warning", "status": "Seen this week", "count": 7, "last_seen": "2026-06-04 19:08", "last_seen_ts": 1780589337.971909, "services": "grafana (7)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780589337971909", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780589337971909", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "KubeNodeEviction", "item_label": "KubeNodeEviction", "severity": "warning", "status": "Seen this week", "count": 4, "last_seen": "2026-06-04 20:50", "last_seen_ts": 1780595443.530579, "services": "grafana (4)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780595443530579", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780595443530579", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "NodeSystemSaturation", "item_label": "NodeSystemSaturation", "severity": "warning", "status": "Seen this week", "count": 3, "last_seen": "2026-06-02 12:28", "last_seen_ts": 1780392529.733539, "services": "grafana (3)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780392529733539", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780392529733539", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TargetDown", "item_label": "TargetDown", "severity": "warning", "status": "Seen this week", "count": 1, "last_seen": "2026-06-01 11:34", "last_seen_ts": 1780302897.768889, "services": "grafana (1)", "signals": "General investigation", "latest_note": "Brief node-exporter target gap caused by Cluster Autoscaler scaling down worker node 10.66.126.144 the node was cordoned, drained, and remo…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780302897768889", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780302897768889?thread_ts=1780302897.768889&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighErrorRate", "item_label": "TraefikServiceHighErrorRate", "severity": "critical", "status": "Recent (72h)", "count": 7, "last_seen": "2026-06-05 09:40", "last_seen_ts": 1780641640.702279, "services": "core-grafana-80 (3), accommodations-api (2), web-80 (1), uni-api (1)", "signals": "General investigation", "latest_note": "@U09JYAWCGLB | uni-api on TUIASI is returning HTTP 500 from StudentServiceImpl.getCatalogCloseStatus and updateCatalogCloseStatus because the JPA query fo…", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780641640702279", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780641640702279?thread_ts=1780641640.702279&cid=C08RDL5L52R", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "application", "source_label": "Application alert", "item_key": "TraefikServiceHighLatency", "item_label": "TraefikServiceHighLatency", "severity": "warning", "status": "Seen today", "count": 21, "last_seen": "2026-06-07 21:40", "last_seen_ts": 1780857608.184429, "services": "ai-api (11), social-api (8), web-80 (7), admission-api (6)", "signals": "No thread signal", "latest_note": "Alert", "alert_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780857608184429", "thread_url": "https://adservio-app.slack.com/archives/C08RDL5L52R/p1780857608184429", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "customer", "source_label": "Customer warning", "item_key": "Adservio Ro", "item_label": "Adservio Ro", "severity": "warning", "status": "Recovered recently", "count": 5, "last_seen": "2026-06-05 18:34", "last_seen_ts": 1780673697.0, "services": "unclassified", "signals": "Pingdom-only evidence so far", "latest_note": "Pingdom-only evidence so far", "alert_url": "http://www.adservio.ro/", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-mysql-master-disk-queue-high", "item_label": "adservio-rds-mysql-master-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-06-06 04:23", "last_seen_ts": 1780709016.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-cpu-high", "item_label": "adservio-rds-postgres-billing-cpu-high", "severity": "warning", "status": "Latest OK", "count": 6, "last_seen": "2026-06-03 11:53", "last_seen_ts": 1780476814.0, "services": "", "signals": "ALARM 3, OK 3, Flips 5", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" }, { "period_start": "2026-05-30T07:00:00+03:00", "period_end": "2026-06-06T07:00:00+03:00", "week_key": "2026-05-30T07:00__2026-06-06T07:00", "week_label": "W22", "source_type": "infrastructure", "source_label": "Infrastructure alarm", "item_key": "adservio-rds-postgres-billing-disk-queue-high", "item_label": "adservio-rds-postgres-billing-disk-queue-high", "severity": "warning", "status": "Latest OK", "count": 2, "last_seen": "2026-06-01 10:13", "last_seen_ts": 1780298027.0, "services": "", "signals": "ALARM 1, OK 1, Flips 1", "latest_note": "OK", "alert_url": "", "thread_url": "", "source_href": "reports/production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.html", "source_json": "production_reliability_dashboard_2026-05-30_0700_2026-06-08_0218.json" } ] }