modules/monitoring: fix probes

machines/biotite/default.nix

@@ -44,6 +44,7 @@
 
   custom.prometheus.exporters = {
     enable = true;
+    node.enable = true;
   };
 
   services.tailscale.enable = true;

machines/thorite/monitoring.nix

@@ -67,10 +67,18 @@ in
       let
         probeList = [
           "la-00.video.namely.icu:8080"
-          "fre-00.video.namely.icu:8080"
+          "fra-00.video.namely.icu:8080"
           "hk-00.video.namely.icu:8080"
           "home.xinyang.life:8000"
         ];
+        chinaTargets = [
+          "bj-cu-v4.ip.zstaticcdn.com:80"
+          "bj-cm-v4.ip.zstaticcdn.com:80"
+          "bj-ct-v4.ip.zstaticcdn.com:80"
+          "sh-cu-v4.ip.zstaticcdn.com:80"
+          "sh-cm-v4.ip.zstaticcdn.com:80"
+          "sh-ct-v4.ip.zstaticcdn.com:80"
+        ];
         passwordFile = config.sops.secrets."prometheus/metrics_password".path;
       in
       (mkScrapes [
@@ -123,6 +131,7 @@ in
         { address = "thorite.coho-tet.ts.net"; }
         { address = "massicot.coho-tet.ts.net"; }
         { address = "weilite.coho-tet.ts.net"; }
+        { address = "biotite.coho-tet.ts.net"; }
         { address = "hk-00.coho-tet.ts.net"; }
         { address = "la-00.coho-tet.ts.net"; }
         { address = "fra-00.coho-tet.ts.net"; }
@@ -140,10 +149,22 @@ in
           hostAddress = "weilite.coho-tet.ts.net";
           targetAddresses = [
             "la-00.video.namely.icu:8080"
-            "fre-00.video.namely.icu:8080"
+            "fra-00.video.namely.icu:8080"
             "hk-00.video.namely.icu:8080"
           ];
         }
+        {
+          hostAddress = "la-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
+        {
+          hostAddress = "hk-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
+        {
+          hostAddress = "fra-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
       ]);
 
   };

overlays/my-lib/prometheus.nix

@@ -237,6 +237,17 @@ in
     {
       inherit name;
       rules = [
+        {
+          alert = "ProbeError";
+          expr = "probe_success != 1";
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Probing {{ $labels.instance }} from {{ $labels.from }} failed";
+          };
+        }
         {
           alert = "HighProbeLatency";
           expr = "probe_duration_seconds > 0.5";