modules/monitoring: clean up tailscale and caddy config

machines/biotite/default.nix

@@ -1,6 +1,7 @@
 {
   pkgs,
   lib,
+  config,
   ...
 }:
 
@@ -41,14 +42,19 @@
     promtail.enable = true;
   };
 
+  custom.prometheus.exporters = {
+    enable = true;
+  };
+
+  services.tailscale.enable = true;
+
+  services.caddy.enable = true;
+
   sops = {
     defaultSopsFile = ./secrets.yaml;
     age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
   };
 
-  services.caddy.enable = true;
-  services.tailscale.enable = true;
-
   services.postgresql = {
     enable = true;
     package = pkgs.postgresql_17;

machines/biotite/services/synapse.nix

@@ -31,6 +31,7 @@ in
 
   services.matrix-synapse = {
     enable = true;
+    withJemalloc = true;
     settings = {
       server_name = "xiny.li";
       public_baseurl = synapseDelegateUrl;

machines/thorite/default.nix

@@ -1,3 +1,4 @@
+{ config, ... }:
 {
   imports = [
     ./hardware-configurations.nix
@@ -30,6 +31,10 @@
       443
     ];
 
+    services.tailscale.enable = true;
+
+    services.caddy.enable = true;
+
     commonSettings = {
       auth.enable = true;
     };

machines/thorite/monitoring.nix

@@ -116,6 +116,8 @@ in
       ])
       ++ (mkCaddyScrapes [
         { address = "thorite.coho-tet.ts.net"; }
+        { address = "biotite.coho-tet.ts.net"; }
+        { address = "weilite.coho-tet.ts.net"; }
       ])
       ++ (mkNodeScrapes [
         { address = "thorite.coho-tet.ts.net"; }

modules/nixos/monitor/default.nix

@@ -82,19 +82,9 @@ in
       ];
     }
     (mkIf cfg.enable {
-
-      services.tailscale = {
-        enable = true;
-        permitCertUid = config.services.caddy.user;
-        openFirewall = true;
-      };
-
-      services.caddy = {
-        enable = true;
-        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
-          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
-        '';
-      };
+      services.caddy.virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+      '';
       services.prometheus = mkIf cfg.enable {
         enable = true;
         port = 9091;

modules/nixos/monitor/exporters.nix

@@ -5,7 +5,8 @@
   ...
 }:
 let
-  inherit (lib) mkIf;
+  inherit (lib) mkIf concatStringsSep;
+  inherit (config.my-lib.settings) prometheusCollectors;
   cfg = config.custom.prometheus.exporters;
 in
 {
@@ -95,8 +96,28 @@ in
         metrics
       }
 
-      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
+      admin unix//var/run/caddy/admin.sock {
+        origins 127.0.0.1 ${config.networking.hostName}.coho-tet.ts.net:2019
       }
     '';
+
+    systemd.services.caddy.serviceConfig = {
+      RuntimeDirectory = "caddy";
+      RuntimeDirectoryMode = "0700";
+    };
+
+    services.tailscale = {
+      permitCertUid = config.services.caddy.user;
+      openFirewall = true;
+    };
+
+    services.caddy = {
+      virtualHosts."https://${config.networking.hostName}.coho-tet.ts.net:2019".extraConfig = ''
+        handle /metrics {
+        	reverse_proxy unix//var/run/caddy/admin.sock
+        }
+        respond 403
+      '';
+    };
   };
 }

modules/nixos/monitor/loki.nix

@@ -241,17 +241,42 @@ in
               ];
             }
             # {
-            #   job_name = "caddy-access";
-            #   file_sd_configs = {
-            #     files = [
-            #       "/var/log/caddy/*.log"
-            #     ];
-            #     refresh_interval = "5m";
-            #   };
+            #   job_name = "caddy";
+            #   static_configs = [
+            #     {
+            #       targets = [ "localhost" ];
+            #       labels = {
+            #         job = "caddy";
+            #         __path__ = "/var/log/caddy/*log";
+            #         agent = "caddy-promtail";
+            #       };
+            #     }
+            #   ];
+            #   pipeline_stages = [
+            #     {
+            #       json = {
+            #         expressions = {
+            #           duration = "duration";
+            #           status = "status";
+            #         };
+            #       };
+            #     }
+            #     {
+            #       labels = {
+            #         duration = null;
+            #         status = null;
+            #       };
+            #     }
+            #   ];
             # }
           ];
         };
       };
+
+      services.caddy.logFormat = ''
+        format json
+        level INFO
+      '';
     })
   ];
 }

note.md

@@ -3,7 +3,8 @@
 Demonstrate disk usage by nix-store path.
 
 ## TODO
-- change caddy admin to unix socket
-- admin config persist = false
-- synapse jmalloc
-- backup all directories under /var/lib/forgejo
+- [x] change caddy admin to unix socket 
+- [ ] admin config persist = false
+- [x] synapse jmalloc
+- [ ] backup all directories under /var/lib/forgejo
+- [ ] collect caddy access logs with promtail (waiting for caddy v2.9.0 release after which log file mode can be set)

overlays/my-lib/prometheus.nix

@@ -28,17 +28,21 @@ in
     )
   );
 
-  mkCaddyScrapes = mkFunction (
+  mkCaddyScrapes = targets: [
     {
-      address,
-      port ? 2019,
-      ...
-    }:
-    {
-      job_name = "caddy_${address}";
-      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+      job_name = "caddy";
+      scheme = "https";
+      static_configs = map (
+        {
+          address,
+          port ? 2019,
+        }:
+        {
+          targets = [ "${address}${mkPort port}" ];
+        }
+      ) targets;
     }
-  );
+  ];
 
   mkCaddyRules = mkFunction (
     {
@@ -63,17 +67,20 @@ in
     }
   );
 
-  mkNodeScrapes = mkFunction (
+  mkNodeScrapes = targets: [
     {
-      address,
-      port ? 9100,
-      ...
-    }:
-    {
-      job_name = "node_${address}";
-      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+      job_name = "node_exporter";
+      static_configs = map (
+        {
+          address,
+          port ? 9100,
+        }:
+        {
+          targets = [ "${address}${mkPort port}" ];
+        }
+      ) targets;
     }
-  );
+  ];
 
   mkNodeRules = mkFunction (
     {

overlays/my-lib/settings.nix

@@ -12,5 +12,9 @@
     grafanaUrl = "https://grafana.xiny.li";
     synapseUrl = "https://xiny.li";
     synapseDelegateUrl = "https://synapse.xiny.li";
+
+    prometheusCollectors = [
+      "thorite.coho-tet.ts.net"
+    ];
   };
 }