monitoring: improve ntfy template

machines/weilite/default.nix

@@ -62,14 +62,6 @@
       defaultSopsFile = ./secrets.yaml;
       age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
       secrets = {
-        cloudflare_dns_token = {
-          owner = "caddy";
-          mode = "400";
-        };
-        dnspod_dns_token = {
-          owner = "caddy";
-          mode = "400";
-        };
         "restic/localpass" = {
           owner = "restic";
         };
@@ -163,38 +155,6 @@
     # tailscale derper module use nginx for reverse proxy
     services.nginx.enable = lib.mkForce false;
 
-    services.caddy = {
-      enable = true;
-      package = pkgs.caddy.withPlugins {
-        plugins = [
-          "github.com/caddy-dns/cloudflare@v0.0.0-20240703190432-89f16b99c18e"
-          "github.com/caddy-dns/dnspod@v0.0.4"
-        ];
-        hash = "sha256-EmBKn6QV5JpLXpez7+Gu91tP/sUZxq2DkGPYoAe+2QM=";
-      };
-      virtualHosts."derper00.namely.icu:8443".extraConfig = ''
-        reverse_proxy 127.0.0.1:${toString config.services.tailscale.derper.port}
-      '';
-      virtualHosts."weilite.coho-tet.ts.net:8080".extraConfig = ''
-        reverse_proxy 127.0.0.1:${toString config.services.immich.port}
-      '';
-      # API Token must be added in systemd environment file
-      virtualHosts."immich.xinyang.life:8000".extraConfig = ''
-        reverse_proxy 127.0.0.1:${toString config.services.immich.port}
-      '';
-      globalConfig = ''
-        acme_dns dnspod {env.DNSPOD_API_TOKEN}
-      '';
-    };
-
-    networking.firewall.allowedTCPPorts = [ 8000 ];
-
-    systemd.services.caddy = {
-      serviceConfig = {
-        EnvironmentFile = config.sops.secrets.dnspod_dns_token.path;
-      };
-    };
-
     time.timeZone = "Asia/Shanghai";
 
     fileSystems."/" = {

machines/weilite/secrets.yaml

@@ -1,5 +1,6 @@
-cloudflare_dns_token: ENC[AES256_GCM,data:m4euSkxxJmiMk9UPyeni/hwpl1W9A4MM0ssg71eOBsX4fFyG39NJeKbNTddW7omBx3gKJtnrRuDdOj5wpg==,iv:eRVzsGwz8hWC42jM+VeSUWCS9Gi8VGSY8Fyh+En0jEI=,tag:NNE8VeNQ8kp9KyziVokyuQ==,type:str]
-dnspod_dns_token: ENC[AES256_GCM,data:uZfr3g103amywxh3NMU+AkwuYb61svzyavvQ4rxJijIMIbfPvERrVNcyivoOrFWYXHpPWkhZFdU=,iv:mArVAcebW9i+u26GmQmfmJTsFkR4ZRMIisTqjpMYan8=,tag:Zsmv1Wzfi3+PHigjReToHQ==,type:str]
+caddy:
+    cf_dns_token: ENC[AES256_GCM,data:7PvP3oYMZ3dAeWaJNiuvEweUf3psDhyu90FT6cP0/AIOa0E40sdIRQ==,iv:IIYnZ35xAm9JJa14oHJi+ddI0u7Pgc4MfPLnKT4IlPc=,tag:V1PGZpaVzdN2cLpktbvTnA==,type:str]
+    dnspod_dns_token: ENC[AES256_GCM,data:ATed7RqLu1u06B61Irhd4SCzjK/Z823ygAgzROsNixZ2rExpB/Xo,iv:L121CGA+iZhn9V6mG2qEu3FI91/s7JO3cVTAwmAeqGw=,tag:l/7MXMZNqgFBwgCCMeZR2A==,type:str]
 immich:
     oauth_client_secret: ENC[AES256_GCM,data:EFs2hPjGMj0idwY3oQVIDTOIWkdwoAoAVjDQE9Z2eAKzUDH3grmYpYE+33V8d/Ux,iv:A9cjwFr/ZqltG62/N8MQ1LhdDbSIVVAqIPVB492zYJw=,tag:VTTtE697BZTVsI32UF53/w==,type:str]
 restic:
@@ -30,8 +31,8 @@ sops:
             V0thRjU4WGpQRGFpcnoxSjZTZHhTTkUKzNMHh9p7GUY3hL5XZ9S4x20CwaItsXFV
             RKujsFVVBd8Kuq/jyOCBTRCscuHI4LW/wYeZYHFEZFSTK2liAqspgw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-25T00:35:15Z"
-    mac: ENC[AES256_GCM,data:sk4DL+w740RD9A3sPvcGD4fc90Nfw9C8dH11ScGRgt6gS3v4V16pD0Q/bHHZiUCll76phZKjp+sGcZaPw0X7RDlK582WY3uw0pLtqLlm0gejjmvBJYKg47nA0dCD+vDvbMkJlvJG6N3sRuXDBa/7bAe452eXZNS8Xnm7ceDscVc=,iv:Nx4yCfG9rNk0q8akuI1aZr6Wj4GIAxASE8Tc7TH4Vj8=,tag:GodvlMbhIPpPu062spKFxA==,type:str]
+    lastmodified: "2025-02-01T15:54:35Z"
+    mac: ENC[AES256_GCM,data:hDX2lQ5GbBGTqioEqNc/k4NvBW7/3ISOVUk8/6CkuW6ZQHUeMnfziWV7faw+DiMvYmwFUJ4mhY77Je5+gid0Ae5JyNxznBW2uzpXvLcTBsYz8iSZL6Jw5FciPIgkGDN5U5wMkusS6Ok2W/idIgmwlmxf3ACNaf7e0QpypwYwxZw=,iv:mkIQ2rvTpQXRuRarlcl/aIKDY3JmJKVsr1oS4+3vmnk=,tag:of2CSCqZAJaaZ5DvC6+Amg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.2

machines/weilite/services/caddy.nix

@@ -0,0 +1,63 @@
+{ config, pkgs, ... }:
+{
+  sops = {
+    secrets = {
+      "caddy/cf_dns_token" = {
+        owner = "caddy";
+        mode = "400";
+      };
+      "caddy/dnspod_dns_token" = {
+        owner = "caddy";
+        mode = "400";
+      };
+    };
+    templates."caddy.env".content = ''
+      CF_API_TOKEN=${config.sops.placeholder."caddy/cf_dns_token"}
+      DNSPOD_API_TOKEN=${config.sops.placeholder."caddy/dnspod_dns_token"}
+    '';
+  };
+
+  services.caddy =
+    let
+      acmeCF = "tls {
+        dns cloudflare {env.CF_API_TOKEN}
+      }";
+      acmeDnspod = "tls {
+        dns dnspod {env.DNSPOD_API_TOKEN}
+      }";
+    in
+    {
+      enable = true;
+      package = pkgs.caddy.withPlugins {
+        plugins = [
+          "github.com/caddy-dns/cloudflare@v0.0.0-20240703190432-89f16b99c18e"
+          "github.com/caddy-dns/dnspod@v0.0.4"
+        ];
+        hash = "sha256-EmBKn6QV5JpLXpez7+Gu91tP/sUZxq2DkGPYoAe+2QM=";
+      };
+      virtualHosts."derper00.namely.icu:8443".extraConfig = ''
+        ${acmeDnspod}
+        reverse_proxy 127.0.0.1:${toString config.services.tailscale.derper.port}
+      '';
+      # API Token must be added in systemd environment file
+      virtualHosts."immich.xinyang.life:8000".extraConfig = ''
+        ${acmeDnspod}
+        reverse_proxy 127.0.0.1:${toString config.services.immich.port}
+      '';
+      virtualHosts."immich.xiny.li:8443".extraConfig = ''
+        ${acmeCF}
+        reverse_proxy 127.0.0.1:${toString config.services.immich.port}
+      '';
+    };
+
+  networking.firewall.allowedTCPPorts = [
+    8000
+    8443
+  ];
+
+  systemd.services.caddy = {
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."caddy.env".path;
+    };
+  };
+}

machines/weilite/services/default.nix

@@ -1,5 +1,6 @@
 {
   imports = [
+    ./caddy.nix
     ./ocis.nix
     ./restic.nix
     ./media-download.nix

machines/weilite/services/restic.nix

@@ -42,6 +42,9 @@ in
   networking.firewall.allowedTCPPorts = [ 8443 ];
 
   services.caddy.virtualHosts."https://backup.xinyang.life:8443".extraConfig = ''
+    tls {
+      dns dnspod {env.DNSPOD_API_TOKEN}
+    }
     reverse_proxy ${config.services.restic.server.listenAddress}
   '';
 }

modules/nixos/monitor/default.nix

@@ -120,11 +120,11 @@ in
                 webhook_configs = [
                   {
                     url = "${ntfyUrl}/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
-                      {{range .alerts}}[{{ if eq .status "resolved" }}✅ RESOLVED{{ else }}{{ if eq .status "firing" }}🔥 FIRING{{end}}{{end}}]{{range $k,$v := .labels}}
-                      {{$k}}={{$v}}{{end}}
-
+                      {{range .alerts}}{{ if eq .status "resolved" }}✅{{ else }}{{ if eq .status "firing" }}🔥{{end}}{{end}}{{.labels.alertname}}
+                      {{.annotations.summary}}
                       {{end}}''}";
                     send_resolved = true;
+                    max_alerts = 5;
                   }
                 ];
               }

overlays/my-lib/prometheus.nix

@@ -1,6 +1,9 @@
 let
   mkFunction = f: (targets: (map f targets));
   mkPort = port: if isNull port then "" else ":${toString port}";
+
+  # get text before "." in the url
+  subdomain = url: builtins.elemAt (builtins.elemAt (builtins.split "([a-zA-Z0-9]+)\..*" url) 1) 0;
 in
 {
   mkScrapes = mkFunction (
@@ -228,7 +231,7 @@ in
       ...
     }:
     {
-      job_name = "blackbox(${hostAddress})";
+      job_name = "blackbox(${subdomain hostAddress})";
       scrape_interval = "1m";
       metrics_path = "/probe";
       params = {
@@ -268,14 +271,14 @@ in
       inherit name;
       rules = [
         {
-          alert = "ProbeError";
-          expr = "probe_success != 1";
+          alert = "ProbeToError";
+          expr = "sum by(instance) (probe_success != 1) > 0";
           for = "3m";
           labels = {
             severity = "critical";
           };
           annotations = {
-            summary = "Probing {{ $labels.instance }} from {{ $labels.from }} failed";
+            summary = "Probing {{ $labels.instance }} failed";
           };
         }
         {