modules/prometheus: fix basic auth scrape

modules/prometheus: add basic auth
modules/prometheus: move to monitor directory
2024-12-02 10:43:43 +08:00 · 2024-11-30 17:35:46 +08:00 · 2024-11-30 12:56:54 +08:00 · 2024-11-30 12:54:51 +08:00 · 2024-11-30 11:25:35 +08:00
32 changed files with 833 additions and 639 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -2,13 +2,22 @@ keys:
  - &xin age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
  - &host-calcite age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
  - &host-raspite age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
-  - &host-sgp-00 age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
  - &host-la-00 age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
  - &host-massicot age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
  - &host-weilite age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml
  - &host-hk-00 age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
  - &host-fra-00 age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+  - &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+  - &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
 creation_rules:
+  - path_regex: machines/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-calcite
+      - *host-massicot
+      - *host-thorite
+      - *host-biotite
  - path_regex: machines/calcite/secrets.yaml
    key_groups:
    - age:
@ -24,19 +33,23 @@ creation_rules:
    - age:
      - *xin
      - *host-massicot
+  - paht_regex: machines/biotite/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-biotite
+  - path_regex: machines/thorite/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-thorite
  - path_regex: machines/dolomite/secrets/secrets.yaml
    key_groups:
    - age:
      - *xin
-      - *host-sgp-00
      - *host-la-00
      - *host-hk-00
      - *host-fra-00
-  - path_regex: machines/dolomite/secrets/sgp-00.yaml
-    key_groups:
-    - age:
-      - *xin
-      - *host-sgp-00
  - path_regex: machines/dolomite/secrets/la-00.yaml
    key_groups:
    - age:
@ -58,16 +71,6 @@ creation_rules:
    - age:
      - *xin
      - *host-weilite
-  - path_regex: machines/secrets.yaml
-    key_groups:
-    - age:
-      - *xin
-      - *host-calcite
-      - *host-raspite
-      - *host-sgp-00
-      - *host-la-00
-      - *host-hk-00
-      - *host-massicot
  - path_regex: home/xin/secrets.yaml
    key_groups:
    - age:
--- a/flake.nix
+++ b/flake.nix
@ -60,8 +60,8 @@
  outputs =
    {
      self,
-      home-manager,
      nixpkgs,
+      home-manager,
      nixos-hardware,
      sops-nix,
      flake-utils,
@ -84,6 +84,7 @@
      overlayModule =
        { ... }:
        {
+          _module.args.my-lib = import ./overlays/my-lib;
          nixpkgs.overlays = [
            editorOverlay
            (import ./overlays/add-pkgs.nix)
@ -118,10 +119,6 @@
          ./machines/dolomite/bandwagon.nix
          ./machines/dolomite/common.nix
        ];
-        tok-00 = [
-          ./machines/dolomite/lightsail.nix
-          ./machines/dolomite/common.nix
-        ];
        fra-00 = [
          ./machines/dolomite/fra.nix
          ./machines/dolomite/common.nix
@ -178,7 +175,6 @@
        };
    in
    {
-      nixpkgs = nixpkgs;
      nixosModules.default = {
        imports = [
          ./modules/nixos
@ -190,7 +186,9 @@
      colmenaHive = colmena.lib.makeHive {
        meta = {
          # FIXME:
-          nixpkgs = import nixpkgs { system = "x86_64-linux"; };
+          nixpkgs = import nixpkgs {
+            system = "x86_64-linux";
+          };
        };

        massicot =
@ -205,20 +203,6 @@
            ] ++ sharedColmenaModules;
          };

-        tok-00 =
-          { ... }:
-          {
-            imports = nodeNixosModules.tok-00 ++ sharedColmenaModules;
-            nixpkgs.system = "x86_64-linux";
-            networking.hostName = "tok-00";
-            system.stateVersion = "23.11";
-            deployment = {
-              targetHost = "video01.namely.icu";
-              buildOnTarget = false;
-              tags = [ "proxy" ];
-            };
-          };
-
        la-00 =
          { ... }:
          {
@ -310,7 +294,6 @@
        osmium = mkNixos {
          hostname = "osmium";
        };
-
      } // self.colmenaHive.nodes;

    }
--- a/machines/biotite/default.nix
+++ b/machines/biotite/default.nix
@ -12,11 +12,12 @@
  networking.useNetworkd = true;
  systemd.network.enable = true;
  systemd.network.networks."10-wan" = {
-    matchConfig.MACAddress = "00:16:3e:0a:ec:45";
-    networkConfig.DHCP = "ipv4";
-    dhcpV4Config = {
-      UseDNS = true;
+    matchConfig.MACAddress = "b6:20:0d:9a:6c:34";
+    networkConfig = {
+      DHCP = "ipv4";
+      IPv6SendRA = true;
    };
+    address = [ "2a03:4000:4a:148::1/64" ];
  };

  commonSettings = {
--- a/machines/calcite/configuration.nix
+++ b/machines/calcite/configuration.nix
@ -368,8 +368,7 @@ in
  };

  custom.prometheus = {
-    enable = true;
-    exporters.blackbox.enable = true;
+    exporters.node.enable = true;
  };

  services.ollama = {
--- a/machines/dolomite/claw.nix
+++ b/machines/dolomite/claw.nix
@ -31,14 +31,7 @@
      fsType = "ext4";
    };

-    swapDevices = [ ];
-
-    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-    # (the default) this is the recommended approach. When using systemd-networkd it's
-    # still possible to use this option, but it's recommended to use it in conjunction
-    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-    # networking.useNetworkd = false;
-
+    networking.useNetworkd = true;
    systemd.network.enable = true;
    systemd.network.networks."10-wan" = {
      matchConfig.MACAddress = "00:16:3e:0a:ec:45";
@ -47,7 +40,6 @@
        UseDNS = true;
      };
    };
-    # networking.interfaces.eth0.useDHCP = lib.mkDefault true;

    nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  };
--- a/machines/dolomite/common.nix
+++ b/machines/dolomite/common.nix
@ -21,12 +21,20 @@
        };
      };
    };
+    swapDevices = [
+      {
+        device = "/swapfile";
+        size = 2 * 1024;
+      }
+    ];

-    custom.prometheus = {
-      enable = lib.mkDefault true;
-      exporters.blackbox.enable = true;
+    custom.prometheus.exporters = {
+      enable = true;
+      node.enable = true;
    };

+    services.tailscale.enable = true;
+
    commonSettings = {
      auth.enable = true;
      proxyServer = {
--- a/machines/dolomite/fra.nix
+++ b/machines/dolomite/fra.nix
@ -52,6 +52,12 @@
    networkConfig = {
      DHCP = "no";
      Gateway = "185.217.108.1";
+      DNSSEC = true;
+      DNSOverTLS = true;
+      DNS = [
+        "8.8.8.8#dns.google"
+        "8.8.4.4#dns.google"
+      ];
    };
    address = [ "185.217.108.59/24" ];
  };
--- a/machines/massicot/default.nix
+++ b/machines/massicot/default.nix
@ -24,10 +24,6 @@
      hedgedoc_env = {
        owner = "hedgedoc";
      };
-      grafana_cloud_api = {
-        owner = "prometheus";
-        sopsFile = ../secrets.yaml;
-      };
      grafana_oauth_secret = {
        owner = "grafana";
      };
@ -62,6 +58,8 @@
    hostName = "massicot";
  };

+  services.tailscale.enable = true;
+
  commonSettings = {
    auth.enable = true;
    nix = {
--- a/machines/massicot/kanidm-provision.nix
+++ b/machines/massicot/kanidm-provision.nix
@ -108,6 +108,20 @@
          ];
        };
      };
+      gotosocial = {
+        displayName = "GoToSocial";
+        originUrl = "https://gts.xiny.li/auth/callback";
+        originLanding = "https://gts.xiny.li/auth/callback";
+        allowInsecureClientDisablePkce = true;
+        scopeMaps = {
+          gts-users = [
+            "openid"
+            "email"
+            "profile"
+            "groups"
+          ];
+        };
+      };
      # It's used for all the clients. I'm too lazy to change the name.
      owncloud-android = {
        displayName = "ownCloud Apps";
@ -147,7 +161,7 @@
      immich = {
        displayName = "Immich";
        originUrl = [
-          "https://immich.xinyang.life:8000/api/oauth/mobile-redirect"
+          "https://immich.xinyang.life:8000/api/oauth/mobile-redirect/"
          "https://immich.xinyang.life:8000/auth/login"
          "https://immich.xinyang.life:8000/user-settings"
        ];
--- a/machines/massicot/networking.nix
+++ b/machines/massicot/networking.nix
@ -3,8 +3,10 @@
  networking.useNetworkd = true;
  systemd.network.networks."10-wan" = {
    matchConfig.MACAddress = "96:00:02:68:7d:2d";
-    networkConfig.DHCP = "ipv4";
-    networkConfig.Gateway = "fe80::1";
+    networkConfig = {
+      DHCP = "ipv4";
+      Gateway = "fe80::1";
+    };
    address = [
      "2a01:4f8:c17:345f::3/64"
    ];
--- a/machines/massicot/services.nix
+++ b/machines/massicot/services.nix
@ -43,10 +43,14 @@ in
    environmentFile = config.sops.secrets.hedgedoc_env.path;
  };

-  custom.prometheus = {
+  custom.prometheus.exporters = {
    enable = true;
-    exporters.blackbox.enable = true;
-    exporters.miniflux.enable = true;
+    blackbox = {
+      enable = true;
+    };
+    node = {
+      enable = true;
+    };
  };

  security.acme = {
--- a/machines/netdrives.nix
+++ b/machines/netdrives.nix
@ -1,22 +0,0 @@
-{ pkgs, config, ... }:
-{
-  sops.secrets = {
-    autofs-nas = {
-      owner = "davfs2";
-    };
-    autofs-nas-secret = {
-      path = "/etc/davfs2/secrets";
-    };
-  };
-  fileSystems."/media/nas" = {
-    device = "https://home.xinyang.life:5244/dav";
-    fsType = "davfs";
-    options = [
-      "uid=1000"
-      "gid=1000"
-      "rw"
-      "_netdev"
-    ];
-
-  };
-}
--- a/machines/secrets.yaml
+++ b/machines/secrets.yaml
@ -1,14 +1,6 @@
-clash_subscription_link: ENC[AES256_GCM,data:uDaX2BE/qRdfXVtckX0VKpu0LN3j0YaxVIPbQt3tGAfdfqFqlp0IzFgNiZBIEcIltYkeEyqFSA0QnttoMb0QYe9f2rtgjztwk10SOGViGaeFWPfkdlHP04qhm5OOOddi3OwT5rUNwvBU79AdCnLJ9QwqMbOaNm/JTtbkcjf8huxc2UcYAQcY/YNJ7aTEhWIw98Ab85aih+w=,iv:pZ189IPPCBjscXzEdgQCRdFlls3TniwDfNCd+H1FFaQ=,tag:dMmGZvppWtkc82b5dTnJwg==,type:str]
-autofs-nas: ENC[AES256_GCM,data:LnCKGKARx6Vd99VwAX/6PXOJwo+a7GP8fNmM9yuuC2xITGxtWCsDdOZL1+IA5LS/gbOYINgQWDzWirJF3LCP27BQeLwXYpD7/UAwwVI=,iv:QJzsS5a6vWeoBxkB13yXdVbyn0tt2QTvqj0LaHn6S2g=,tag:D/JKXQIw1EzIh3wjGhHgHg==,type:str]
-autofs-nas-secret: ENC[AES256_GCM,data:gbOizRZAvh79HlJWIWeKTk79Ux311XGL1eIswc0P2U2huCibD/ji3kOlSjZXENG+fJQKNz2AlDTk3g2cQQ==,iv:UCaGeE8j4RqJzA0xhu3oB2xvzombzQD3fjLKCWd5fDg=,tag:II4eEMr7f2TDUl1qUcDYXA==,type:str]
-github_public_token: ENC[AES256_GCM,data:6Gt+oJcCRHeoLK7CRndMMbszTXSEbnN0nQzsVOnl/+zB4hxbEPD5k/vkkl+cZ/qmxdxFXV0OOsYvktn44Yv1DMUE3mkB0hcAdoyPwLuYM7W3RpOoW3OktH8DRCUi6msvFp3ykpdmIl9WyjVhc/lMwTaYJQyRh1ue,iv:PJSFtJBelyc3rzd6hqjMp+ciU2Q3FTOEXsiq5F2KKTY=,tag:Y/stRg6kwyjjIFZCXS/peg==,type:str]
-singbox_sg_server: ENC[AES256_GCM,data:SF2ja6W4TwThwoug5x2KTA==,iv:Vx9wNTdVHkReux4YeQY+0VkC1Wqg/CRkY7frVY/3e50=,tag:7XA9KSoR0GA6FoYRhCv4BQ==,type:str]
-singbox_jp_server: ENC[AES256_GCM,data:S3Bs5yVMzyz6vD51GYElOM5h,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:o9d55cZuWmX4NDYexWjvYQ==,type:str]
-sing-box:
-    password: ENC[AES256_GCM,data:xyqmoJEDI5959zHPTVelln/iThtoeDwS,iv:rLyqJsE/4JDf08RlMLLPh+MKJkba9bL0z8jx6bTEfgc=,tag:cgLHdeLIyPvLhRNaVcQ0TQ==,type:str]
-    uuid: ENC[AES256_GCM,data:lWBCM5wyz6BcUUHdvynkn5y166Kk15jO0EhWUDuhXXhrve5l,iv:RmDJYFnYqIEIShLn25sf4h8AO2E3+3Xa2U9Mff+Xk2w=,tag:SN0DUdwZXKO/VEnozrr5mA==,type:str]
-grafana_cloud_api: ENC[AES256_GCM,data:eEvPAwtThK1FMhbrnmSo89+GlWZAF+LQRMLXA2C6f1vR7ZPlXJZGWzjYwDcPlnpiC737/cG14M4kZqvPGBuNub5A83rBS/+FeebvGDIF59L5PC1Ys1jWBB9YRI/L9EU0tvwTTUCvLRA9j28n7Jw7wR6mWXm63XA+OMu8/UbTwbeV/WUQn8vnwqadSUdCnNKJXMsAY+q9t/st0DPm5+aNxA==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:87C+0FVvzDIowE0+QpY1zA==,type:str]
-private_dns_address: ENC[AES256_GCM,data:YJxNOH4hsZHResvANEqJRTANhnL4PLp/Pmi/PhgtSTbTKiJKPqudhTEkNg==,iv:8+qG5rQXAKfrykEjt9qrbtyNaBuKvi7EaIWouRqEipY=,tag:VH0w5ZbXcWFGZ9GLavm7/w==,type:str]
+prometheus:
+    metrics_username: ENC[AES256_GCM,data:/CQfOA==,iv:BjhB+uLfjmYHdgpc/+tDJXJ8C1EK9kngQWbo4NleOmE=,tag:JCdqyqGLRh09T25vmufiZw==,type:str]
+    metrics_password: ENC[AES256_GCM,data:q/xMPuNtlcUFewMdVu6w2Q==,iv:xLohdb5tdxevYFckZoacjSJp2rZ53QKLxK6u3mc3mDw=,tag:B4LrObH1DsnnD5CcuOPOyg==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -18,77 +10,50 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiRzZVNGFocUN2VzZLTmJz
-            WlJnUmxhZS92citDRkVZVnJZQU9YWVZORlNjCkgzeWl5dTl1YmpjZGt2anF3dGgr
-            K1hOSTRmakNrZ2JoNit3NDIzK1FCcWMKLS0tIEdqY3VvR3gxd1JoQlhPR3JvcXBF
-            K2g4VFpqUEF5RTQ3cmpUSG0xajN2bUUKMuwx5cO1nHokV1NOloXfl9wTBN/+/Rlq
-            UJKP/qaI23tpyMXN1U40iF20ecO1U5Ad8wAQ61C/tldSVULizDihpw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmUUtKcU0va1F3SjFzcktl
+            S0FJNHlTUzBkMWlydGRjM1Y4SGd2SXpMVTJRCmtRSW9wNW9xMDBaQ0YzUWM4YjRz
+            SVRDNHRjNG5hTHBOOHorTTlJU1BwY1EKLS0tIEpLREJ1VzFaalczZlhKaitHVTJU
+            MDdJaVBtVmw4WTlBUEF5WXJSVFRFeDAKnvF6CmnU8hxXSdKQPUJqPT7Dewl4REOH
+            wDQELRaDkMPMKEOAc6wCmXNErvj/I7w7wuvB5WxtanC7g4IEphD6aA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYXlwdytVSm1SQzRubHdX
-            dHhrTWxyamo3OFRraEVRQ3plK1cwUWt0a1JRCkdqaVRTQ0NaTkdoMlpDT3Yyallq
-            eTd0bDViVTgwZGRTUmlYTzR0Y09iWWcKLS0tIEFlQnFPVFRVNlAxdExMekJ2b295
-            UUJkUUZCNUZnbkNFZHVBYXNHQklOL2MKujgh6REuAKu6ZLVA7atiWUqhnvYJnQjb
-            WsxCa9ZXZRgfbhcNlZ3qIKJpWWI/RMS17+Nm5yIl+2cSqe2UJMjZdg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNXQ4RVVRd3RYRkhUVExG
-            SHJON0hwSmJtUkYrd1dldHJRN2tPKzBsNlNjCi9xYVhaanF5TDU2Q0xadXNWR2tN
-            dHhQVkpRREFlRm5MM2pwVytEaGhHT0EKLS0tIE9sRUtLako3cnAxNm82RDhiWEVM
-            ZW1IMXkzYkhqbW1ZdVRabUlkK2oxSTgKHC22uQqMq+cJ7vrONkGgoH8snxGef6Ft
-            QbtoJziERjAhK6B7TOY8AJ3WVRpCzZN70HjLNYa+bMMNOvmlsVxfZg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSmZ5YUpFdzRNdWZVNmxJ
-            bm5ucUhVeTV2TkE4ZElkZ1N3aXc2eVEwMlRNCmxXRElPb2pGYzJFVnUxQkRtMlNF
-            cjgwUzh5UWNLTk01U0h2bHNpaXVzZkkKLS0tIDczUkkwTG8rL1V3UU9lenk4V2tl
-            TUxDd2huTllMRG9MZTJZdzRwaWxqUVEKLA3y+heUA8cK31LZzv5A1wtgf+sauuwE
-            7SGU3uYU650tJM3e6Lveo+JOAD7Z1jrAomT5Bub+jjSHnpeFC9yMbA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4enA2bnlrV3ViY1RHaTVS
-            Ym5VV005NFlXZUl2NDNXYXBoOHh0SGQ0YVNnCm1KdHBSeE9lQzZEM2hFZUwzRitS
-            K3BEWGhtWmxKc2RJd0FTMEs2b1ArOUkKLS0tIG5kaWc3U0o4SG9teXk5dVZWWjkz
-            cS9VMU5YbEl3UE5mODJ1THNLVEdVblEKNQF0b9r1XPD819Z6Uy0b9hT4Uek2tNWU
-            3z3H7V/UiB1TMW+qgs6BC6bDkDf7oG//qmZEdYF+lDXcNSwai25xyg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBreHJNRXlpOEh6YUxaSmJj
-            ZDlVdHh2b1p6aEs3eDAvbkk4WExxWmE5bDM0CkZzT2l1K09UbmNFNEpZUVY2NVlB
-            dVFYbnpvTjlUcTdZejMrelpscXRJQzAKLS0tIEVIaVByVmp0aUU1ZWJLajBhcjRk
-            QVZMRXBRVVhaY3JKZEJjMTdEeEVqcWMKT+DoevNQAxCrty2VkRDLWGFzs9GsW3F7
-            txz73tAceAIiocC1z7IV2TaYULYf7Z75HAje/SOTlGHBIDiVZ0vyLw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBS2xqbWp0NkFMVm1Cci82
+            d0ZmRXdRc2FFSklHNnZtV1Z0REFIelZDQ1U4CllmdllNVnp3WmpCeDNBRzVFbVR5
+            WkJFMGs5ZWJEK1lSWDQwYUdOdFJseGcKLS0tIEZUU254aWtYdWthL3I2UkJ0eklj
+            WHhrRlRvLzlmY0REYktGSlh3MENzRzgKzO1XqXhcXAxfn86+IY+ccBII1SGYctAk
+            +ArpGmXaf53RFmPLSzMGNaiJzfhqk9U9bn3WV9CFdaA7Rtec0ZAcNw==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEZEJHQlJqMWxob1lxOUFK
-            dDZIN3FaNWR1L0gyN3I1MXVXZlpzdlpQUHpjCmIwTWhRamZvSTF3cHZMNk9YUlRv
-            U2tOK3E5MFBFNERsUHVzVnhsUDFRd00KLS0tIFd1MUpaaFU0bWdVRjJ6NjFwcFZt
-            bkJGWFFWanFBK1drZlBNcHo0c3Bjc00K/vPBLocRhtcJ3snGYFr+H7qhbg6iSSPP
-            OSH8WnaM5JmmA9IQlm5uGiG74PHi5sg5d+bwG8pPQtMKN+Ndxh7JIA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycmppSGR1YTMzTTJQSmx0
+            eFlRQWpPRTR1L042MXlqd3dURStjaUZjR1M4Ci9VdzZkSmN3d1BDNTlJdXNSenhZ
+            MDE1VktBN252L2FYMjJmNEFITGptM3MKLS0tIDNYOEZqNjM1VCtEWDlGdzYveG5j
+            dEp2bmVQMmV5ZU9Jb3FFSDFoT2NJOE0Kx1ZifyU2WLoHeUmqP9oCUmIl6ZJeytGB
+            WPMJKcNtuJHL1OWhT0wMiv6NEF5UaYXIlCqSVtXAMy554G4JlX5tQw==
            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
+        - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlN291MzZOaU4zazhEeXBh
-            WlhoYmh2ZDBsZmc3cEthdW5paWpXbXQvUG1FCjBLZ0FPVWR3T2pVWTZrRmkxSWUr
-            MHhkUFFPK1Z0b2t1Z1J0VjlER1JvcGMKLS0tIE45YndxVW4vak1wcEJoZzhHQ0E0
-            NzA1cy80ZW5vUFplQzVMZ0txSmVkMUEKFUvgmJNdo9sV33gOx7LVUSCYvIqCNwaP
-            u+XoWTfg4kp9f4KVTy/8huPsVLhZBUaf6jI10mV2z4QwaLHje4JiHw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5OGlRWVM0Z01pYjdqYjEy
+            TGdhNFhpNEUwSFZaeHNwRkpraFF5RTU2SXhnCm5YWmM5SmdERzZBWTgva1E4MDFm
+            N0xyUExGV0MvbFF3M0ZRSVEydFNUSGMKLS0tIGxxNWhsSEt4WDR2a2hId1JkVFE3
+            enJ6MzJxR0I4eStSQk9ON0dsdjFmRkEKBSGkv1O0vgHSsU3+6AGN7bKQ5lpN7AMT
+            eqEgWx7juZ7hKzLq1HMbiT61l0FrJNHEMfn15bzn7GsK5YJQvfiq9w==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-22T05:48:59Z"
-    mac: ENC[AES256_GCM,data:In/gSIYnXKbbv1lzS/nmSESCHBcBv/TtkvhzdNiIn73N4kP9aJ+1JE8Npix8zNItzk46DX+nHBk8Kwgl6uq26YtL+sMTBKh5K8Ny0H8ivlgS+olXswv3Y9h1cYD7FBHUKzbMuiJd0ppjC0ZIn20rRpb4d57rwUbvY0KstyQW4JA=,iv:DcdTAimbXXpKhhiB9rriS75+XGNOCcScqi/804+Xx6g=,tag:NHW+UViRmbUDHb0gTd9TDg==,type:str]
+        - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeVlxRk1PZUpUVm1xckRu
+            ZmMzdHdvKzExaTQxVHYvYjIvblo3b1ZORFJrCkpVdHFLbCtNS0xnamJ0T250YzUy
+            Uy9Xd0tMa3FSVlRkQXFaTWJVem9uWGsKLS0tIFRmT0VzL0hlLzkrRTZxcWtLN3Qv
+            YVMya3dUazFyaWRNNDJ3OVNIVXJLVTQK+7MxkmBjPszozXUO+zVaWdsovDmhWAfz
+            8puIpXpWZY09BkS0vs4oNhiVA9PD11TBIVCEbC5E1TwpwboMXBYhCQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-30T06:31:42Z"
+    mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/sops.nix
+++ b/machines/sops.nix
@ -10,20 +10,9 @@
      # TODO: How to generate this key when bootstrap?
      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
      secrets = {
-        github_public_token = {
-          owner = "root";
-        };
-        singbox_sg_server = {
-          owner = "root";
-        };
-        singbox_jp_server = {
-          owner = "root";
-        };
-        private_dns_address = {
-          owner = "root";
-        };
+        "prometheus/metrics_username" = { };
+        "prometheus/metrics_password" = { };
      };
-      secrets.grafana_cloud_api = lib.mkIf config.services.prometheus.enable { owner = "prometheus"; };
    };
  };
 }
--- a/machines/thorite/default.nix
+++ b/machines/thorite/default.nix
@ -1,29 +1,40 @@
-{ ... }:
 {
  imports = [
    ./hardware-configurations.nix
+    ./monitoring.nix
  ];

-  networking.hostName = "thorite";
-  networking.useNetworkd = true;
-  systemd.network.enable = true;
-  systemd.network.networks."10-wan" = {
-    matchConfig.MACAddress = "00:51:d3:21:f3:28";
-    networkConfig = {
-      DHCP = "no";
-      Gateway = "23.165.200.1";
+  config = {
+    networking.hostName = "thorite";
+    networking.useNetworkd = true;
+    systemd.network.enable = true;
+    systemd.network.networks."10-wan" = {
+      matchConfig.MACAddress = "00:51:d3:21:f3:28";
+      networkConfig = {
+        DHCP = "no";
+        Gateway = "23.165.200.1";
+        DNSSEC = true;
+        DNSOverTLS = true;
+        DNS = [
+          "8.8.8.8#dns.google"
+          "8.8.4.4#dns.google"
+        ];
+      };
+      address = [ "23.165.200.99/24" ];
    };
-    address = [ "23.165.200.99/24" ];
+
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+
+    commonSettings = {
+      auth.enable = true;
+    };
+
+    nixpkgs.system = "x86_64-linux";
+    system.stateVersion = "24.11";
+
+    users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
  };
-
-  nixpkgs.system = "x86_64-linux";
-
-  system.stateVersion = "24.11";
-
-  commonSettings = {
-    auth.enable = true;
-    autoupgrade.enable = true;
-  };
-
-  users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
 }
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@ -0,0 +1,103 @@
+{ config, my-lib, ... }:
+with my-lib;
+{
+  config = {
+    sops = {
+      defaultSopsFile = ./secrets.yaml;
+      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+      secrets = {
+        "grafana/oauth_secret" = {
+          owner = "grafana";
+        };
+      };
+    };
+
+    custom.monitoring = {
+      grafana.enable = true;
+    };
+
+    services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =
+      with config.services.grafana.settings.server; ''
+        reverse_proxy http://${http_addr}:${toString http_port}
+      '';
+
+    custom.prometheus = {
+      enable = true;
+      exporters = {
+        enable = true;
+        blackbox.enable = true;
+        node.enable = true;
+      };
+      ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]);
+    };
+
+    services.prometheus.scrapeConfigs =
+      let
+        probeList = [
+          "la-00.video.namely.icu:8080"
+          "fre-00.video.namely.icu:8080"
+          "hk-00.video.namely.icu:8080"
+          "49.13.13.122:443"
+          "45.142.178.32:22"
+          "home.xinyang.life:8000"
+        ];
+        passwordFile = config.sops.secrets."prometheus/metrics_password".path;
+      in
+      (mkScrapes [
+        {
+          name = "immich";
+          scheme = "http";
+          address = "weilite.coho-tet.ts.net";
+          port = 8082;
+        }
+        {
+          inherit passwordFile;
+          name = "gotosocial";
+          address = "xinyang.life";
+        }
+        {
+          inherit passwordFile;
+          name = "miniflux";
+          address = "rss.xinyang.life";
+        }
+        {
+          name = "ntfy";
+          address = "ntfy.xinyang.life";
+        }
+        {
+          name = "grafana-eu";
+          address = "grafana.xinyang.life";
+        }
+      ])
+      ++ (mkCaddyScrapes [
+        { address = "thorite.coho-tet.ts.net"; }
+      ])
+      ++ (mkNodeScrapes [
+        { address = "thorite.coho-tet.ts.net"; }
+        { address = "massicot.coho-tet.ts.net"; }
+        { address = "weilite.coho-tet.ts.net"; }
+        { address = "hk-00.coho-tet.ts.net"; }
+        { address = "la-00.coho-tet.ts.net"; }
+        { address = "fra-00.coho-tet.ts.net"; }
+      ])
+      ++ (mkBlackboxScrapes [
+        {
+          hostAddress = "thorite.coho-tet.ts.net";
+          targetAddresses = probeList;
+        }
+        {
+          hostAddress = "massicot.coho-tet.ts.net";
+          targetAddresses = probeList;
+        }
+        {
+          hostAddress = "weilite.coho-tet.ts.net";
+          targetAddresses = [
+            "la-00.video.namely.icu:8080"
+            "fre-00.video.namely.icu:8080"
+            "hk-00.video.namely.icu:8080"
+          ];
+        }
+      ]);
+
+  };
+}
--- a/machines/thorite/secrets.yaml
+++ b/machines/thorite/secrets.yaml
@ -0,0 +1,31 @@
+grafana:
+    oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoTXRtTlRES3V4MGhZaGdr
+            aXJ4UFNDT0Nrb0ZuWEkxUEFDU2orbzNBSVhVCkh2VitqMGwwOVdhMFJIeWU1eTgw
+            UVdxY0tLVDJNVnRnQmMyS0FPYS9LVmMKLS0tIEZaMTdIMU5SQUkxL2NFK2Jtbm9v
+            YVR3RHpDR3F2aFlCWGd5TjNOV2p4YzgK8OKpwcvTK/0j+kQCo0+8n6sQ5Pu9t9xZ
+            lPWeUGk1BudsyCqgIZWF5iXfu1pJnYq1XEAM0ttJl402xKeqIovM0Q==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIYk1NTlhsYW8xbFppNTBE
+            WlJmNzhnclBoVENXa1cvcHY5NGdRZVAzV0FJClpsTHpTeG9CK3J2ZFEreG1BTWpG
+            WjdaYzlLQnU0LzJLSDBZZ2pvOWdvSEkKLS0tIExRT0p1aCttZG5MMW14emJmRk5w
+            M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx
+            mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-28T17:02:03Z"
+    mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.1
--- a/machines/weilite/default.nix
+++ b/machines/weilite/default.nix
@ -16,7 +16,6 @@
    networking.hostName = "weilite";
    commonSettings = {
      auth.enable = true;
-      autoupgrade.enable = true;
      nix = {
        enable = true;
        enableMirrors = true;
@ -61,8 +60,14 @@
      };
    };

-    custom.prometheus = {
+    custom.prometheus.exporters = {
      enable = true;
+      blackbox = {
+        enable = true;
+      };
+      node = {
+        enable = true;
+      };
    };

    systemd.mounts = [
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -7,7 +7,7 @@
    ./disk-partitions
    ./restic.nix
    ./vaultwarden.nix
-    ./prometheus
+    ./monitor
    ./hedgedoc.nix
    ./sing-box.nix
    ./kanidm-client.nix
--- a/modules/nixos/monitor/default.nix
+++ b/modules/nixos/monitor/default.nix
@ -0,0 +1,171 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    mkIf
+    mkMerge
+    types
+    ;
+  cfg = config.custom.prometheus;
+
+  mkRulesOption = mkOption {
+    type = types.listOf (
+      types.submodule {
+        options = {
+          name = mkOption { type = lib.types.str; };
+          rules = mkOption { type = lib.types.listOf lib.types.attrs; };
+        };
+      }
+    );
+  };
+in
+{
+  imports = [
+    ./exporters.nix
+    ./grafana.nix
+  ];
+
+  options = {
+    custom.monitoring = {
+      grafana = {
+        enable = mkEnableOption "grafana with oauth only";
+      };
+    };
+    custom.prometheus = {
+      enable = mkEnableOption "Prometheus instance";
+      ruleModules = mkRulesOption;
+      exporters = {
+        enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
+        node = {
+          enable = mkEnableOption "node exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+        blackbox = {
+          enable = mkEnableOption "blackbox exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+      };
+    };
+  };
+
+  config = mkMerge [
+    {
+      sops.secrets = {
+        "prometheus/metrics_username" = {
+          sopsFile = ../../../machines/secrets.yaml;
+          group = "prometheus-auth";
+          mode = "0440";
+        };
+
+        "prometheus/metrics_password" = {
+          sopsFile = ../../../machines/secrets.yaml;
+          group = "prometheus-auth";
+          mode = "0440";
+        };
+      };
+
+      users.groups.prometheus-auth.members = [
+        "prometheus"
+      ];
+    }
+    (mkIf cfg.enable {
+
+      services.tailscale = {
+        enable = true;
+        permitCertUid = config.services.caddy.user;
+        openFirewall = true;
+      };
+
+      services.caddy = {
+        enable = true;
+        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+        '';
+      };
+      services.prometheus = mkIf cfg.enable {
+        enable = true;
+        port = 9091;
+        globalConfig.external_labels = {
+          hostname = config.networking.hostName;
+        };
+
+        scrapeConfigs = [
+          {
+            job_name = "prometheus";
+            static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
+          }
+        ];
+
+        alertmanager = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          logLevel = "debug";
+          configuration = {
+            route = {
+              receiver = "ntfy";
+            };
+            receivers = [
+              {
+                name = "ntfy";
+                webhook_configs = [
+                  {
+                    url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
+                      Alert {{.status}}
+                      {{range .alerts}}-----{{range $k,$v := .labels}}
+                      {{$k}}={{$v}}{{end}}
+                      {{end}}
+                    ''}";
+                    send_resolved = true;
+                  }
+                ];
+              }
+            ];
+          };
+        };
+
+        alertmanagers = [
+          {
+            scheme = "http";
+            static_configs = [
+              {
+                targets = [
+                  "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
+                ];
+              }
+            ];
+          }
+        ];
+        rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
+      };
+      custom.prometheus.ruleModules = [
+        {
+          name = "prometheus_alerts";
+          rules = [
+            {
+              alert = "JobDown";
+              expr = "up == 0";
+              for = "1m";
+              labels = {
+                severity = "critical";
+              };
+              annotations = {
+                summary = "Job {{ $labels.job }} down for 1m.";
+              };
+            }
+          ];
+        }
+      ];
+    })
+  ];
+}
--- a/modules/nixos/monitor/exporters.nix
+++ b/modules/nixos/monitor/exporters.nix
@ -0,0 +1,102 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkIf;
+  cfg = config.custom.prometheus.exporters;
+in
+{
+  config = {
+    systemd.services.tailscaled.after =
+      (lib.optional cfg.node.enable "prometheus-node-exporters.service")
+      ++ (lib.optional cfg.blackbox.enable "prometheus-blackbox-exporters.service")
+      ++ (lib.optional config.services.caddy.enable "caddy.service");
+
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      enabledCollectors = [
+        "loadavg"
+        "time"
+        "systemd"
+      ];
+      listenAddress = cfg.node.listenAddress;
+      port = 9100;
+    };
+
+    services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
+      enable = true;
+      listenAddress = cfg.blackbox.listenAddress;
+      configFile = pkgs.writeText "blackbox.config.yaml" (
+        lib.generators.toYAML { } {
+          modules = {
+            tcp4_connect = {
+              prober = "tcp";
+              tcp = {
+                ip_protocol_fallback = false;
+                preferred_ip_protocol = "ip4";
+                tls = false;
+              };
+              timeout = "15s";
+            };
+          };
+        }
+      );
+    };
+
+    # gotosocial
+    sops.templates."gotosocial_metrics.env" = {
+      content = ''
+        GTS_METRICS_AUTH_ENABLED=true
+        GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
+        GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
+      '';
+      group = "prometheus-auth";
+      mode = "0440";
+    };
+    systemd.services.gotosocial.serviceConfig = {
+      EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ];
+      SupplementaryGroups = [ "prometheus-auth" ];
+    };
+
+    services.gotosocial.settings = {
+      metrics-enabled = true;
+    };
+
+    services.immich.environment = {
+      IMMICH_TELEMETRY_INCLUDE = "all";
+    };
+
+    services.restic.server.prometheus = true;
+
+    # miniflux 
+    sops.templates."miniflux_metrics_env" = {
+      content = ''
+        METRICS_COLLECTOR=1
+        LOG_LEVEL=debug
+        METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
+        METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
+      '';
+      group = "prometheus-auth";
+      mode = "0440";
+    };
+
+    systemd.services.miniflux.serviceConfig = {
+      EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ];
+      SupplementaryGroups = [ "prometheus-auth" ];
+    };
+
+    services.ntfy-sh.settings.enable-metrics = true;
+
+    services.caddy.globalConfig = ''
+      servers {
+        metrics
+      }
+
+      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
+      }
+    '';
+  };
+}
--- a/modules/nixos/monitor/grafana.nix
+++ b/modules/nixos/monitor/grafana.nix
@ -0,0 +1,43 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.monitoring.grafana;
+in
+{
+  config = lib.mkIf cfg.enable {
+    sops.templates."grafana.env".content = ''
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${config.sops.placeholder."grafana/oauth_secret"}
+    '';
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3003;
+          root_url = "https://grafana.xinyang.life";
+          domain = "grafana.xinyang.life";
+        };
+        "auth.generic_oauth" = {
+          enabled = true;
+          name = "Kanidm";
+          client_id = "grafana";
+          scopes = "openid,profile,email,groups";
+          auth_url = "https://auth.xinyang.life/ui/oauth2";
+          token_url = "https://auth.xinyang.life/oauth2/token";
+          api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
+          use_pkce = true;
+          use_refresh_token = true;
+          allow_sign_up = true;
+          login_attribute_path = "preferred_username";
+          groups_attribute_path = "groups";
+          role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
+          allow_assign_grafana_admin = true;
+          auto_login = true;
+        };
+        "auth" = {
+          disable_login_form = true;
+        };
+      };
+    };
+    systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.templates."grafana.env".path;
+  };
+}
--- a/modules/nixos/prometheus/restic.nix
+++ b/modules/nixos/prometheus/restic.nix
@ -3,17 +3,26 @@ let
  cfg = config.custom.prometheus;
 in
 {
-  config = lib.mkIf (cfg.enable && cfg.exporters.restic.enable) {
+  config = {
    services.restic.server.prometheus = true;

-    services.prometheus.scrapeConfigs = [
-      (lib.mkIf cfg.exporters.restic.enable {
-        job_name = "restic";
-        static_configs = [ { targets = [ config.services.restic.server.listenAddress ]; } ];
-      })
-    ];
+    custom.prometheus.templates.scrape.mkResticScrapes =
+      {
+        address,
+        port ? null,
+        ...
+      }:
+      let
+        portStr = if port then ":${toString port}" else "";
+      in
+      [
+        (lib.mkIf cfg.exporters.restic.enable {
+          job_name = "restic";
+          static_configs = [ { targets = [ "${address}${portStr}" ]; } ];
+        })
+      ];

-    custom.prometheus.ruleModules = [
+    custom.prometheus.templates.rules.mkResticRules = [
      {
        name = "restic_alerts";
        rules = [
--- a/modules/nixos/prometheus/blackbox.nix
+++ b/modules/nixos/prometheus/blackbox.nix
@ -1,93 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.blackbox.enable) {
-    services.prometheus.exporters.blackbox = {
-      enable = true;
-      listenAddress = "127.0.0.1";
-      configFile = pkgs.writeText "blackbox.config.yaml" (
-        lib.generators.toYAML { } {
-          modules = {
-            tcp4_connect = {
-              prober = "tcp";
-              tcp = {
-                ip_protocol_fallback = false;
-                preferred_ip_protocol = "ip4";
-                tls = false;
-              };
-              timeout = "15s";
-            };
-          };
-        }
-      );
-    };
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "blackbox";
-        scrape_interval = "1m";
-        metrics_path = "/probe";
-        params = {
-          module = [ "tcp4_connect" ];
-        };
-        static_configs = [
-          {
-            targets = [
-              "tok-00.namely.icu:8080"
-              "la-00.video.namely.icu:8080"
-              "auth.xinyang.life:443"
-              "home.xinyang.life:8000"
-            ];
-          }
-        ];
-        relabel_configs = [
-          {
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox_exporter";
-        static_configs = [
-          { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }
-        ];
-      }
-    ];
-
-    custom.prometheus.ruleModules = [
-      {
-        name = "probe_alerts";
-        rules = [
-          {
-            alert = "HighProbeLatency";
-            expr = "probe_duration_seconds > 0.5";
-            for = "2m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High request latency on {{ $labels.instance }}";
-              description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes.";
-            };
-          }
-        ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/caddy.nix
+++ b/modules/nixos/prometheus/caddy.nix
@ -1,40 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.caddy.enable) {
-    services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable ''
-      servers {
-        metrics
-      }
-    '';
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "caddy";
-        static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ];
-      }
-    ];
-
-    custom.prometheus.ruleModules = [
-      {
-        name = "caddy_alerts";
-        rules = [
-          {
-            alert = "UpstreamHealthy";
-            expr = "caddy_reverse_proxy_upstreams_healthy != 1";
-            for = "5m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Upstream {{ $labels.unstream }} not healthy";
-            };
-          }
-        ];
-      }
-    ];
-  };
-
-}
--- a/modules/nixos/prometheus/default.nix
+++ b/modules/nixos/prometheus/default.nix
@ -1,230 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-
-with lib;
-
-let
-  cfg = config.custom.prometheus;
-  mkExporterOption =
-    enableOption:
-    (mkOption {
-      type = types.bool;
-      default = enableOption;
-      description = "Enable this exporter";
-    });
-
-  mkRulesOption = mkOption {
-    type = types.listOf (
-      types.submodule {
-        options = {
-          name = mkOption { type = lib.types.str; };
-          rules = mkOption { type = lib.types.listOf lib.types.attrs; };
-        };
-      }
-    );
-  };
-in
-{
-  imports = [
-    ./blackbox.nix
-    ./caddy.nix
-    ./gotosocial.nix
-    ./immich.nix
-    ./miniflux.nix
-    ./ntfy-sh.nix
-    ./restic.nix
-  ];
-
-  options = {
-    custom.prometheus = {
-      enable = mkEnableOption "Prometheus instance";
-      exporters = {
-        enable = mkOption {
-          type = types.bool;
-          default = false;
-          description = "Enable Prometheus exporter on every supported services";
-        };
-
-        restic.enable = mkExporterOption config.services.restic.server.enable;
-        blackbox.enable = mkExporterOption false;
-        caddy.enable = mkExporterOption config.services.caddy.enable;
-        gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
-        immich.enable = mkExporterOption config.services.immich.enable;
-        miniflux.enable = mkExporterOption config.services.miniflux.enable;
-        ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable;
-      };
-      grafana = {
-        enable = mkEnableOption "Grafana Cloud";
-        password_file = mkOption { type = types.path; };
-      };
-      ruleModules = mkRulesOption;
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.tailscale = {
-      enable = true;
-      permitCertUid = config.services.caddy.user;
-      openFirewall = true;
-    };
-
-    services.caddy = {
-      enable = true;
-      virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
-        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
-      '';
-    };
-
-    services.prometheus = mkIf cfg.enable {
-      enable = true;
-      port = 9091;
-      globalConfig.external_labels = {
-        hostname = config.networking.hostName;
-      };
-      remoteWrite = mkIf cfg.grafana.enable [
-        {
-          name = "grafana";
-          url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
-          basic_auth = {
-            username = "1340065";
-            password_file = cfg.grafana.password_file;
-          };
-        }
-      ];
-      exporters = {
-        node = {
-          enable = true;
-          enabledCollectors = [
-            "loadavg"
-            "time"
-            "systemd"
-          ];
-          listenAddress = "127.0.0.1";
-          port = 9100;
-        };
-      };
-      scrapeConfigs = [
-        {
-          job_name = "prometheus";
-          static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
-        }
-        {
-          job_name = "node";
-          static_configs = [
-            { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
-          ];
-        }
-      ];
-
-      alertmanager = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        logLevel = "debug";
-        configuration = {
-          route = {
-            receiver = "ntfy";
-          };
-          receivers = [
-            {
-              name = "ntfy";
-              webhook_configs = [
-                {
-                  url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
-                    Alert {{.status}}
-                    {{range .alerts}}-----{{range $k,$v := .labels}}
-                    {{$k}}={{$v}}{{end}}
-                    {{end}}
-                  ''}";
-                  send_resolved = true;
-                }
-              ];
-            }
-          ];
-        };
-      };
-
-      alertmanagers = [
-        {
-          scheme = "http";
-          static_configs = [
-            {
-              targets = [
-                "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
-              ];
-            }
-          ];
-        }
-      ];
-      rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
-    };
-    custom.prometheus.ruleModules = [
-      {
-        name = "system_alerts";
-        rules = [
-          {
-            alert = "SystemdFailedUnits";
-            expr = "node_systemd_unit_state{state=\"failed\"} > 0";
-            for = "5m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Systemd has failed units on {{ $labels.instance }}";
-              description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
-            };
-          }
-          {
-            alert = "HighLoadAverage";
-            expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
-            for = "1m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High load average detected on {{ $labels.instance }}";
-              description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
-            };
-          }
-          {
-            alert = "HighTransmitTraffic";
-            expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
-            for = "1m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
-              description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
-            };
-          }
-          {
-            alert = "NetworkTrafficExceedLimit";
-            expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
-            for = "0m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Outbound network traffic exceed 300GB for last 30 day";
-            };
-          }
-          {
-            alert = "JobDown";
-            expr = "up == 0";
-            for = "1m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Job {{ $labels.job }} down for 1m.";
-            };
-          }
-        ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/gotosocial.nix
+++ b/modules/nixos/prometheus/gotosocial.nix
@ -1,17 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.gotosocial.enable) {
-    services.gotosocial.settings = {
-      metrics-enabled = true;
-    };
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "gotosocial";
-        static_configs = [ { targets = [ "localhost:8080" ]; } ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/immich.nix
+++ b/modules/nixos/prometheus/immich.nix
@ -1,25 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-  immichEnv = config.services.immich.environment;
-  metricPort =
-    if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv then
-      immichEnv.IMMICH_API_METRICS_PORT
-    else
-      8081;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) {
-    services.immich.environment = {
-      IMMICH_METRICS = "true";
-    };
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "immich";
-        static_configs = [ { targets = [ "127.0.0.1:${toString metricPort}" ]; } ];
-      }
-    ];
-  };
-
-}
--- a/modules/nixos/prometheus/miniflux.nix
+++ b/modules/nixos/prometheus/miniflux.nix
@ -1,15 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.miniflux.enable) {
-    systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "miniflux";
-        static_configs = [ { targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; } ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/ntfy-sh.nix
+++ b/modules/nixos/prometheus/ntfy-sh.nix
@ -1,15 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.ntfy-sh.enable) {
-    services.ntfy-sh.settings.enable-metrics = true;
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "ntfy-sh";
-        static_configs = [ { targets = [ "ntfy.xinyang.life" ]; } ];
-      }
-    ];
-  };
-}
--- a/overlays/my-lib/default.nix
+++ b/overlays/my-lib/default.nix
@ -0,0 +1,3 @@
+{
+}
+// (import ./prometheus.nix)
--- a/overlays/my-lib/prometheus.nix
+++ b/overlays/my-lib/prometheus.nix
@ -0,0 +1,209 @@
+let
+  mkFunction = f: (targets: (map f targets));
+  mkPort = port: if isNull port then "" else ":${toString port}";
+in
+{
+  mkScrapes = mkFunction (
+    {
+      name,
+      address,
+      passwordFile ? null,
+      port ? 443,
+      scheme ? "https",
+      ...
+    }:
+    {
+      job_name = "${name}(${address})";
+      scheme = scheme;
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+    // (
+      if isNull passwordFile then
+        { }
+      else
+        {
+          basic_auth.username = "prom";
+          basic_auth.password_file = passwordFile;
+        }
+    )
+  );
+
+  mkCaddyScrapes = mkFunction (
+    {
+      address,
+      port ? 2019,
+      ...
+    }:
+    {
+      job_name = "caddy_${address}";
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+  );
+
+  mkCaddyRules = mkFunction (
+    {
+      host ? "",
+      name ? "caddy_alerts_${host}",
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "UpstreamHealthy";
+          expr = "caddy_reverse_proxy_upstreams_healthy != 1";
+          for = "5m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Upstream {{ $labels.unstream }} not healthy";
+          };
+        }
+      ];
+    }
+  );
+
+  mkNodeScrapes = mkFunction (
+    {
+      address,
+      port ? 9100,
+      ...
+    }:
+    {
+      job_name = "node_${address}";
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+  );
+
+  mkNodeRules = mkFunction (
+    {
+      host ? "",
+      name ? "system_alerts_${host}",
+      ...
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "SystemdFailedUnits";
+          expr = "node_systemd_unit_state{state=\"failed\"} > 0";
+          for = "5m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Systemd has failed units on {{ $labels.instance }}";
+            description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
+          };
+        }
+        {
+          alert = "HighLoadAverage";
+          expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
+          for = "1m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High load average detected on {{ $labels.instance }}";
+            description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
+          };
+        }
+        {
+          alert = "HighTransmitTraffic";
+          expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
+          for = "1m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
+            description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
+          };
+        }
+        {
+          alert = "NetworkTrafficExceedLimit";
+          expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
+          for = "0m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Outbound network traffic exceed 300GB for last 30 day";
+          };
+        }
+      ];
+    }
+  );
+
+  mkBlackboxScrapes = mkFunction (
+    {
+      hostAddress,
+      hostPort ? 9115,
+      targetAddresses,
+      ...
+    }:
+    {
+      job_name = "blackbox(${hostAddress})";
+      scrape_interval = "1m";
+      metrics_path = "/probe";
+      params = {
+        module = [ "tcp4_connect" ];
+      };
+      static_configs = [
+        {
+          targets = targetAddresses;
+        }
+      ];
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          source_labels = [ "__param_target" ];
+          target_label = "instance";
+        }
+        {
+          target_label = "__address__";
+          replacement = "${hostAddress}${mkPort hostPort}";
+        }
+      ];
+    }
+  );
+
+  mkBlackboxRules = mkFunction (
+    {
+      host ? "",
+      name ? "probe_alerts_${host}",
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "HighProbeLatency";
+          expr = "probe_duration_seconds > 0.5";
+          for = "3m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High request latency on {{ $labels.instance }}";
+            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+          };
+        }
+        {
+          alert = "VeryHighProbeLatency";
+          expr = "probe_duration_seconds > 1";
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "High request latency on {{ $labels.instance }}";
+            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+          };
+        }
+      ];
+    }
+  );
+}
Author	SHA1	Message	Date
xinyangli	92db38383e	modules/prometheus: fix basic auth scrape	2024-12-02 10:43:43 +08:00
xinyangli	74b67e1854	modules/prometheus: add basic auth	2024-11-30 17:35:46 +08:00
xinyangli	97fcdefc2b	modules/prometheus: move to monitor directory	2024-11-30 12:56:54 +08:00
xinyangli	9e3af9a535	modules/prometheus: start exporters after tailscaled	2024-11-30 12:54:51 +08:00
xinyangli	ec63c8b6cc	modules/prometheus: refactor	2024-11-30 11:25:35 +08:00