feat: enable tailscale on servers

modules/nixos/prometheus.nix

@@ -26,6 +26,18 @@ in
   };
 
   config = mkIf cfg.enable (mkMerge [{
+    services.tailscale = {
+      enable = true;
+      permitCertUid = config.services.caddy.user;
+    };
+
+    services.caddy = {
+      enable = true;
+      virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+      '';
+    };
+
     services.caddy.globalConfig = ''
       servers {
         metrics
@@ -103,7 +115,7 @@ in
               name = "ntfy";
               webhook_configs = [
                 {
-                  url = "${config.services.ntfy-sh.settings.base-url}/prometheus-alerts";
+                  url = "https://ntfy.xinyang.life/prometheus-alerts";
                   send_resolved = true;
                 }
               ];
@@ -162,7 +174,55 @@ in
           ''
             groups:
               - name: restic_alerts
+                rules:
+                  - alert: ResticCheckFailed
+                    expr: restic_check_success == 0
+                    for: 5m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: Restic check failed (instance {{ $labels.instance }})
+                      description: Restic check failed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}
+
+                  - alert: ResticOutdatedBackup
+                    # 1209600 = 15 days
+                    expr: time() - restic_backup_timestamp > 518400
+                    for: 0m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated
+                      description: Restic backup is outdated\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}
           '' else "")
+        (if config.services.caddy.enable then ''
+            groups:
+              - name: caddy_alerts
+                rules:
+                  - alert: HighHttpErrorRate
+                    expr: rate(caddy_http_request_duration_seconds_count{status_code=~"5.."}[5m]) / rate(caddy_http_request_duration_seconds_count[5m]) > 0.01
+                    for: 10m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: "High error rate on {{ $labels.instance }}"
+                      description: "More than 1% of HTTP requests are errors over the last 10 minutes."
+                  - alert: CaddyDown
+                    expr: up{job="caddy"} == 0
+                    for: 5m
+                    labels:
+                      severity: critical
+                    annotations:
+                      summary: "Caddy server down on {{ $labels.instance }}"
+                      description: "Caddy server is down for more than 5 minutes."
+                  - alert: HighRequestLatency
+                    expr: histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5
+                    for: 2m
+                    labels:
+                      severity: warning
+                    annotations:
+                      summary: "High request latency on {{ $labels.instance }}"
+                      description: "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."
+        '' else "")
       ];
     };
   }