From e4fd9e8b234d41f5f0a6508cb142ad84604bd006 Mon Sep 17 00:00:00 2001
From: xinyangli <lixinyang411@gmail.com>
Date: Fri, 6 Dec 2024 23:25:44 +0800
Subject: [PATCH] modules/monitoring: fix probes

---
 machines/biotite/default.nix    |  1 +
 machines/thorite/monitoring.nix | 25 +++++++++++++++++++++++--
 overlays/my-lib/prometheus.nix  | 11 +++++++++++
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/machines/biotite/default.nix b/machines/biotite/default.nix
index 5a51ab0..741e281 100644
--- a/machines/biotite/default.nix
+++ b/machines/biotite/default.nix
@@ -44,6 +44,7 @@
 
   custom.prometheus.exporters = {
     enable = true;
+    node.enable = true;
   };
 
   services.tailscale.enable = true;
diff --git a/machines/thorite/monitoring.nix b/machines/thorite/monitoring.nix
index e9cbb3b..dd3b693 100644
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@@ -67,10 +67,18 @@ in
       let
         probeList = [
           "la-00.video.namely.icu:8080"
-          "fre-00.video.namely.icu:8080"
+          "fra-00.video.namely.icu:8080"
           "hk-00.video.namely.icu:8080"
           "home.xinyang.life:8000"
         ];
+        chinaTargets = [
+          "bj-cu-v4.ip.zstaticcdn.com:80"
+          "bj-cm-v4.ip.zstaticcdn.com:80"
+          "bj-ct-v4.ip.zstaticcdn.com:80"
+          "sh-cu-v4.ip.zstaticcdn.com:80"
+          "sh-cm-v4.ip.zstaticcdn.com:80"
+          "sh-ct-v4.ip.zstaticcdn.com:80"
+        ];
         passwordFile = config.sops.secrets."prometheus/metrics_password".path;
       in
       (mkScrapes [
@@ -123,6 +131,7 @@ in
         { address = "thorite.coho-tet.ts.net"; }
         { address = "massicot.coho-tet.ts.net"; }
         { address = "weilite.coho-tet.ts.net"; }
+        { address = "biotite.coho-tet.ts.net"; }
         { address = "hk-00.coho-tet.ts.net"; }
         { address = "la-00.coho-tet.ts.net"; }
         { address = "fra-00.coho-tet.ts.net"; }
@@ -140,10 +149,22 @@ in
           hostAddress = "weilite.coho-tet.ts.net";
           targetAddresses = [
             "la-00.video.namely.icu:8080"
-            "fre-00.video.namely.icu:8080"
+            "fra-00.video.namely.icu:8080"
             "hk-00.video.namely.icu:8080"
           ];
         }
+        {
+          hostAddress = "la-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
+        {
+          hostAddress = "hk-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
+        {
+          hostAddress = "fra-00.coho-tet.ts.net";
+          targetAddresses = chinaTargets;
+        }
       ]);
 
   };
diff --git a/overlays/my-lib/prometheus.nix b/overlays/my-lib/prometheus.nix
index b7607a1..3642d55 100644
--- a/overlays/my-lib/prometheus.nix
+++ b/overlays/my-lib/prometheus.nix
@@ -237,6 +237,17 @@ in
     {
       inherit name;
       rules = [
+        {
+          alert = "ProbeError";
+          expr = "probe_success != 1";
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Probing {{ $labels.instance }} from {{ $labels.from }} failed";
+          };
+        }
         {
           alert = "HighProbeLatency";
           expr = "probe_duration_seconds > 0.5";