From 947e97ce4e9c510165b94bed15e62f5aec19fb3e Mon Sep 17 00:00:00 2001
From: xinyangli <lixinyang411@gmail.com>
Date: Tue, 3 Dec 2024 19:35:48 +0800
Subject: [PATCH] thorite: fix loki alerts
---
machines/thorite/monitoring.nix | 6 +++---
modules/nixos/monitor/loki.nix | 16 +++++++++-------
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/machines/thorite/monitoring.nix b/machines/thorite/monitoring.nix
index 164776e..d9901e3 100644
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@@ -18,11 +18,11 @@ with my-lib;
enable = true;
rules = {
sshd_closed = {
- condition = ''count_over_time({unit="sshd.service"} |~ "Connection closed by authenticating user" [15m]) > 25'';
- description = "More then 25 users have tried logging in the last 15 min without success";
+ expr = ''count_over_time({unit="sshd.service"} |~ "Connection closed by authenticating user" [15m]) > 25'';
+ description = "More then 25 login attemps in last 15 min without success";
};
unusual_log_volume = {
- condition = ''sum by (unit) (rate({unit=~".+"}[5m])) > 80'';
+ expr = ''sum by (unit) (rate({unit=~".+"}[5m])) > 80'';
description = "Unit {{ $labels.unit }} is logging at an unusually high rate";
};
};
diff --git a/modules/nixos/monitor/loki.nix b/modules/nixos/monitor/loki.nix
index c3e0afd..772d59a 100644
--- a/modules/nixos/monitor/loki.nix
+++ b/modules/nixos/monitor/loki.nix
@@ -29,7 +29,7 @@ in
type = types.attrsOf (
types.submodule {
options = {
- condition = mkOption {
+ expr = mkOption {
type = types.str;
description = ''
Loki alert expression.
@@ -85,7 +85,7 @@ in
name = "alerting-rules";
rules = lib.mapAttrsToList (name: opts: {
alert = name;
- inherit (opts) condition labels;
+ inherit (opts) expr labels;
for = opts.time;
annotations.description = opts.description;
}) cfg.loki.rules;
@@ -137,20 +137,22 @@ in
ruler = {
storage = {
type = "local";
- local.directory = "${config.services.loki.dataDir}/ruler";
+ local.directory = "${config.services.loki.dataDir}/rules";
};
- rule_path = "${config.services.loki.dataDir}/rules";
+ rule_path = "${config.services.loki.dataDir}/rules-temp";
+ enable_api = true;
alertmanager_url = "http://127.0.0.1:${toString alertmanagerPort}";
};
};
};
systemd.tmpfiles.rules = [
"d /var/lib/loki 0700 loki loki - -"
- "d /var/lib/loki/ruler 0700 loki loki - -"
+ "d /var/lib/loki/rules-temp 0700 loki loki - -"
"d /var/lib/loki/rules 0700 loki loki - -"
- "L /var/lib/loki/ruler/ruler.yml - - - - ${rulerFile}"
+ "d /var/lib/loki/rules/fake 0700 loki loki - -"
+ "L /var/lib/loki/rules/fake/ruler.yml - - - - ${rulerFile}"
];
- systemd.services.loki.reloadTriggers = [ rulerFile ];
+ systemd.services.loki.restartTriggers = [ rulerFile ];
}
)
(mkIf cfg.promtail.enable {