modules/monitoring: clean up tailscale and caddy config

biotite: move all services to biotite except kanidm
modules: clean up
2024-12-05 17:36:01 +08:00 · 2024-12-05 13:28:22 +08:00 · 2024-12-04 19:54:32 +08:00 · 2024-12-04 18:19:22 +08:00 · 2024-12-04 16:34:51 +08:00 · 2024-12-04 16:34:10 +08:00
49 changed files with 1250 additions and 1176 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -15,6 +15,7 @@ creation_rules:
    - age:
      - *xin
      - *host-calcite
+      - *host-weilite
      - *host-massicot
      - *host-thorite
      - *host-biotite
--- a/flake.lock
+++ b/flake.lock
@ -2,11 +2,11 @@
  "nodes": {
    "catppuccin": {
      "locked": {
-        "lastModified": 1731232837,
-        "narHash": "sha256-0aIwr/RC/oe7rYkfJb47xjdEQDSNcqpFGsEa+EPlDEs=",
+        "lastModified": 1733001911,
+        "narHash": "sha256-uX/9m0TbdhEzuWA0muM5mI/AaWcLiDLjCCyu5Qr9MRk=",
        "owner": "catppuccin",
        "repo": "nix",
-        "rev": "32359bf226fe874d3b7a0a5753d291a4da9616fe",
+        "rev": "a817009ebfd2cca7f70a77884e5098d0a8c83f8e",
        "type": "github"
      },
      "original": {
@ -68,11 +68,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1732645828,
-        "narHash": "sha256-+4U2I2653JvPFxcux837ulwYS864QvEueIljUkwytsk=",
+        "lastModified": 1732988076,
+        "narHash": "sha256-2uMaVAZn7fiyTUGhKgleuLYe5+EAAYB/diKxrM7g3as=",
        "owner": "nix-community",
        "repo": "disko",
-        "rev": "869ba3a87486289a4197b52a6c9e7222edf00b3e",
+        "rev": "2814a5224a47ca19e858e027f7e8bff74a8ea9f1",
        "type": "github"
      },
      "original": {
@ -238,11 +238,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730302582,
-        "narHash": "sha256-W1MIJpADXQCgosJZT8qBYLRuZls2KSiKdpnTVdKBuvU=",
+        "lastModified": 1730814269,
+        "narHash": "sha256-fWPHyhYE6xvMI1eGY3pwBTq85wcy1YXqdzTZF+06nOg=",
        "owner": "cachix",
        "repo": "git-hooks.nix",
-        "rev": "af8a16fe5c264f5e9e18bcee2859b40a656876cf",
+        "rev": "d70155fdc00df4628446352fc58adc640cd705c2",
        "type": "github"
      },
      "original": {
@ -281,11 +281,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731786860,
-        "narHash": "sha256-130gQ5k8kZlxjBEeLpE+SvWFgSOFgQFeZlqIik7KgtQ=",
+        "lastModified": 1733085484,
+        "narHash": "sha256-dVmNuUajnU18oHzBQWZm1BQtANCHaqNuxTHZQ+GN0r8=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "1bd5616e33c0c54d7a5b37db94160635a9b27aeb",
+        "rev": "c1fee8d4a60b89cae12b288ba9dbc608ff298163",
        "type": "github"
      },
      "original": {
@ -303,11 +303,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730490306,
-        "narHash": "sha256-AvCVDswOUM9D368HxYD25RsSKp+5o0L0/JHADjLoD38=",
+        "lastModified": 1731235328,
+        "narHash": "sha256-NjavpgE9/bMe/ABvZpyHIUeYF1mqR5lhaep3wB79ucs=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "1743615b61c7285976f85b303a36cdf88a556503",
+        "rev": "60bb110917844d354f3c18e05450606a435d2d10",
        "type": "github"
      },
      "original": {
@ -332,16 +332,16 @@
        ]
      },
      "locked": {
-        "lastModified": 1729544999,
-        "narHash": "sha256-YcyJLvTmN6uLEBGCvYoMLwsinblXMkoYkNLEO4WnKus=",
+        "lastModified": 1729958008,
+        "narHash": "sha256-EiOq8jF4Z/zQe0QYVc3+qSKxRK//CFHMB84aYrYGwEs=",
        "owner": "NuschtOS",
        "repo": "ixx",
-        "rev": "65c207c92befec93e22086da9456d3906a4e999c",
+        "rev": "9fd01aad037f345350eab2cd45e1946cc66da4eb",
        "type": "github"
      },
      "original": {
        "owner": "NuschtOS",
-        "ref": "v0.0.5",
+        "ref": "v0.0.6",
        "repo": "ixx",
        "type": "github"
      }
@ -355,11 +355,11 @@
        "nixvim": "nixvim"
      },
      "locked": {
-        "lastModified": 1730642581,
-        "narHash": "sha256-Tcq+RnctJTm+TUr1fN3ivqYNcd1pJnHYzLDQdgUCX70=",
+        "lastModified": 1732936640,
+        "narHash": "sha256-NcluA0L+ZV5MUj3UuQhlkGCj8KoEhX/ObWlMHZ/F/ac=",
        "ref": "refs/heads/master",
-        "rev": "a09d2b94efb5e2d801275a244eedaab0816f3702",
-        "revCount": 18,
+        "rev": "a3709a89797ea094f82d38edeb4a538c07c8c3fa",
+        "revCount": 20,
        "type": "git",
        "url": "https://git.xinyang.life/xin/nixvim"
      },
@ -377,11 +377,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730448474,
-        "narHash": "sha256-qE/cYKBhzxHMtKtLK3hlSR3uzO1pWPGLrBuQK7r0CHc=",
+        "lastModified": 1731153869,
+        "narHash": "sha256-3Ftf9oqOypcEyyrWJ0baVkRpvQqroK/SVBFLvU3nPuc=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "683d0c4cd1102dcccfa3f835565378c7f3cbe05e",
+        "rev": "5c74ab862c8070cbf6400128a1b56abb213656da",
        "type": "github"
      },
      "original": {
@ -418,11 +418,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731814505,
-        "narHash": "sha256-l9ryrx1Twh08a+gxrMGM9O/aZKEimZfa6sZVyPCImgI=",
+        "lastModified": 1733024876,
+        "narHash": "sha256-vy9Q41hBE7Zg0yakF79neVgb3i3PQMSMR7uHPpPywFE=",
        "owner": "Mic92",
        "repo": "nix-index-database",
-        "rev": "bdba246946fb079b87b4cada4df9b1cdf1c06132",
+        "rev": "6e0b7f81367069589a480b91603a10bcf71f3103",
        "type": "github"
      },
      "original": {
@ -442,11 +442,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731808759,
-        "narHash": "sha256-WwJqguc/5Q7HEwHlgDzDT8mtd8ZxInxZM2neJKC1oh8=",
+        "lastModified": 1733104664,
+        "narHash": "sha256-UhlyYYO84s36aSj0/xZdclY6CgwJSWPYtTHTOBuHodM=",
        "owner": "nix-community",
        "repo": "nix-vscode-extensions",
-        "rev": "5cf92678e6799ce45442dee4c9cb8094843c7cfa",
+        "rev": "e3a9b717e8327886d4ab6115f6989f4d1ef44e51",
        "type": "github"
      },
      "original": {
@ -457,11 +457,11 @@
    },
    "nixos-hardware": {
      "locked": {
-        "lastModified": 1731797098,
-        "narHash": "sha256-UhWmEZhwJZmVZ1jfHZFzCg+ZLO9Tb/v3Y6LC0UNyeTo=",
+        "lastModified": 1733066523,
+        "narHash": "sha256-aQorWITXZu7b095UwnpUvcGt9dNJie/GO9r4hZfe2sU=",
        "owner": "NixOS",
        "repo": "nixos-hardware",
-        "rev": "672ac2ac86f7dff2f6f3406405bddecf960e0db6",
+        "rev": "fe01780d356d70fd119a19277bff71d3e78dad00",
        "type": "github"
      },
      "original": {
@ -473,11 +473,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1730200266,
-        "narHash": "sha256-l253w0XMT8nWHGXuXqyiIC/bMvh1VRszGXgdpQlfhvU=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "807e9154dcb16384b1b765ebe9cd2bba2ac287fd",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
        "type": "github"
      },
      "original": {
@ -501,11 +501,11 @@
    },
    "nixpkgs-stable": {
      "locked": {
-        "lastModified": 1731652201,
-        "narHash": "sha256-XUO0JKP1hlww0d7mm3kpmIr4hhtR4zicg5Wwes9cPMg=",
+        "lastModified": 1733016324,
+        "narHash": "sha256-8qwPSE2g1othR1u4uP86NXxm6i7E9nHPyJX3m3lx7Q4=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "c21b77913ea840f8bcf9adf4c41cecc2abffd38d",
+        "rev": "7e1ca67996afd8233d9033edd26e442836cc2ad6",
        "type": "github"
      },
      "original": {
@ -515,29 +515,13 @@
        "type": "github"
      }
    },
-    "nixpkgs-stable_2": {
-      "locked": {
-        "lastModified": 1731797254,
-        "narHash": "sha256-df3dJApLPhd11AlueuoN0Q4fHo/hagP75LlM5K1sz9g=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "e8c38b73aeb218e27163376a2d617e61a2ad9b59",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1731819057,
-        "narHash": "sha256-nfqKsQhFCakM+eIKGf/JWu/g56rOPoGny10EZN8q7R0=",
+        "lastModified": 1733128666,
+        "narHash": "sha256-JOIhbU0EPRXwFv1wCXGTkUZ9KnIcLxChvCqeV9hh63U=",
        "owner": "xinyangli",
        "repo": "nixpkgs",
-        "rev": "b2644ed7258502987ad4a70cf8959bf5a26ce26d",
+        "rev": "6273ca0a0fd51ac708a71e380c0cda97a72bbb07",
        "type": "github"
      },
      "original": {
@ -560,11 +544,11 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1730569492,
-        "narHash": "sha256-NByr7l7JetL9kIrdCOcRqBu+lAkruYXETp1DMiDHNQs=",
+        "lastModified": 1731527733,
+        "narHash": "sha256-12OpSgbLDiKmxvBXwVracIfGI9FpjFyHpa1r0Ho+NFA=",
        "owner": "nix-community",
        "repo": "nixvim",
-        "rev": "6f210158b03b01a1fd44bf3968165e6da80635ce",
+        "rev": "f11a877bcc1d66cc8bd7990c704f91c1e99c7d08",
        "type": "github"
      },
      "original": {
@ -575,11 +559,11 @@
    },
    "nur": {
      "locked": {
-        "lastModified": 1731819675,
-        "narHash": "sha256-GGp/rEfxRdi1BD9TlHoXxp2g9IuKDp0Jk7wYh1LacP8=",
+        "lastModified": 1733125101,
+        "narHash": "sha256-C8f6ekiZ4kP84JWLDrMigvnSK6RXQoxLEDoteXMx1yc=",
        "owner": "nix-community",
        "repo": "NUR",
-        "rev": "59740d792bea5caa547c9bc7ce366802ecfafb7f",
+        "rev": "1844924bf1e7e5a98198eca17b6c27cc9a363b05",
        "type": "github"
      },
      "original": {
@ -599,11 +583,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730515563,
-        "narHash": "sha256-8lklUZRV7nwkPLF3roxzi4C2oyLydDXyAzAnDvjkOms=",
+        "lastModified": 1731060242,
+        "narHash": "sha256-43yLsOm/wxBbfYSNDWVJeVv5Ij+23X3BIjFUfsdx/6M=",
        "owner": "NuschtOS",
        "repo": "search",
-        "rev": "9e22bd742480916ff5d0ab20ca2522eaa3fa061e",
+        "rev": "ef493352f9e1f051e01a55c062731503a6b36b4e",
        "type": "github"
      },
      "original": {
@ -633,15 +617,14 @@
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
-        ],
-        "nixpkgs-stable": "nixpkgs-stable_2"
+        ]
      },
      "locked": {
-        "lastModified": 1731814239,
-        "narHash": "sha256-TGnMXCeXS924w9W6CvRFtUCUFr8E/RK138lHxU3vcw8=",
+        "lastModified": 1733128155,
+        "narHash": "sha256-m6/qwJAJYcidGMEdLqjKzRIjapK4nUfMq7rDCTmZajc=",
        "owner": "Mic92",
        "repo": "sops-nix",
-        "rev": "47fc1d8c72dbd69b32ecb2019b5b648da3dd20ce",
+        "rev": "c6134b6fff6bda95a1ac872a2a9d5f32e3c37856",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -84,11 +84,16 @@
      overlayModule =
        { ... }:
        {
-          _module.args.my-lib = import ./overlays/my-lib;
-          nixpkgs.overlays = [
-            editorOverlay
-            (import ./overlays/add-pkgs.nix)
-          ];
+          options.my-lib = nixpkgs.lib.mkOption {
+            type = nixpkgs.lib.types.attrs;
+            default = import ./overlays/my-lib;
+          };
+          config = {
+            nixpkgs.overlays = [
+              editorOverlay
+              (import ./overlays/add-pkgs.nix)
+            ];
+          };
        };
      deploymentModule = {
        deployment.targetUser = "xin";
@ -109,7 +114,7 @@
          nur.nixosModules.nur
          catppuccin.nixosModules.catppuccin
          machines/calcite/configuration.nix
-          (mkHome "xin" "calcite")
+          # (mkHome "xin" "calcite")
        ];
        hk-00 = [
          ./machines/dolomite/claw.nix
@ -142,17 +147,15 @@
        user: host:
        { ... }:
        {
-          imports = [
-            home-manager.nixosModules.home-manager
-            {
-              home-manager = {
-                sharedModules = sharedHmModules;
-                useGlobalPkgs = true;
-                useUserPackages = true;
-              };
-              home-manager.users.${user} = (import ./home).${user}.${host};
-            }
-          ];
+          imports = [ home-manager.nixosModules.home-manager ];
+          config = {
+            home-manager = {
+              sharedModules = sharedHmModules;
+              useGlobalPkgs = true;
+              useUserPackages = true;
+            };
+            home-manager.users.${user} = (import ./home).${user}.${host};
+          };
        };
      mkNixos =
        {
--- a/home/xin/calcite.nix
+++ b/home/xin/calcite.nix
@ -125,7 +125,8 @@ in
    profiles.default = {
      isDefault = true;
      userChrome = ''
-        #titlebar {
+
+        #TabsToolbar {
          display: none;
        }

@ -136,7 +137,7 @@ in
        [titlepreface*="."] #sidebar-header {
          visibility: collapse !important;
        }
-        [titlepreface*="."] #titlebar {
+        [titlepreface*="."] #TabsToolbar {
          visibility: collapse;
        }

@ -148,7 +149,7 @@ in
          min-width: var(--uc-sidebar-width) !important;
          width: var(--uc-sidebar-width) !important;
          max-width: var(--uc-sidebar-width) !important;
-          z-index:1;
+          z-index: calc(var(--browser-area-z-index-tabbox) + 1);
        }

        #sidebar-box[positionend]{ direction: rtl }
@ -190,12 +191,12 @@ in
          transition-delay: 0ms !important;
        }

-        .sidebar-panel{
-          background-color: transparent !important;
+        .sidebar-placeTree {
+          /* background-color: transparent !important; */
          color: var(--newtab-text-primary-color) !important;
        }

-        .sidebar-panel #search-box{
+        .sidebar-placeTree #search-box{
          -moz-appearance: none !important;
          background-color: rgba(249,249,250,0.1) !important;
          color: inherit !important;
--- a/machines/biotite/default.nix
+++ b/machines/biotite/default.nix
@ -1,5 +1,7 @@
 {
+  pkgs,
  lib,
+  config,
  ...
 }:

@ -7,6 +9,12 @@
  imports = [
    ./hardware-configurations.nix
    ./services/gotosocial.nix
+    ./services/synapse.nix
+    ./services/restic.nix
+    ./services/miniflux.nix
+    ./services/hedgedoc.nix
+    ./services/forgejo.nix
+    ./services/vaultwarden.nix
  ];

  networking.hostName = "biotite";
@ -28,20 +36,55 @@

  commonSettings = {
    auth.enable = true;
-    autoupgrade.enable = true;
  };

  custom.monitoring = {
    promtail.enable = true;
  };

+  custom.prometheus.exporters = {
+    enable = true;
+  };
+
+  services.tailscale.enable = true;
+
+  services.caddy.enable = true;
+
  sops = {
    defaultSopsFile = ./secrets.yaml;
    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
  };

-  services.caddy.enable = true;
-  services.tailscale.enable = true;
+  services.postgresql = {
+    enable = true;
+    package = pkgs.postgresql_17;
+    settings = {
+      allow_alter_system = false;
+      # DB Version: 17
+      # OS Type: linux
+      # DB Type: mixed
+      # Total Memory (RAM): 8 GB
+      # CPUs num: 4
+      # Data Storage: ssd
+      max_connections = 100;
+      shared_buffers = "2GB";
+      effective_cache_size = "6GB";
+      maintenance_work_mem = "512MB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 200;
+      work_mem = "5242kB";
+      huge_pages = "off";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 4;
+      max_parallel_workers_per_gather = 2;
+      max_parallel_workers = 4;
+      max_parallel_maintenance_workers = 2;
+    };
+  };

  users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";

--- a/machines/biotite/secrets.yaml
+++ b/machines/biotite/secrets.yaml
@ -1,5 +1,16 @@
 gotosocial:
    oidc_client_secret: ENC[AES256_GCM,data:KVQxzs67sohax2h0Y/jjhnbY4fetrdVvWhBGbqgDSGgBC7QazrOmTA++BSRzMmVv,iv:HIRMc56aLanqQRTWH9E0wzzXymImi0pxK/ccPEP8Fcc=,tag:PMhOLeE3mKIIQveRdfpgpA==,type:str]
+synapse:
+    oidc_client_secret: ENC[AES256_GCM,data:TdZF8Bo+h34fn03sPpt7JEqmP8Cwm8V++q9VDvaapMBc3rlkrVu3iDUhQE2DvJri,iv:/QNX+aYUPpDKIqWZ13TLAznR3ZpUPI8rQHrJuqv7R+g=,tag:lcBIpeWiIXK/NV84uuxNiA==,type:str]
+restic:
+    repo_url: ENC[AES256_GCM,data:ZcBMqwEsyc7zyEftJZj4XkKBzUHwlqd6cjX8xVDn9m26jBL7aP5atpnXDRE9FXY4CuAllFyQZyAOQ2L61Nfx+iplL2ADbSoH,iv:fhNODiyoOlZEqYR2O/GsH2IWTPDr3rXSJgWC/EFDLSA=,tag:nZdKKnpiszSiXxdZI1KQ/A==,type:str]
+    repo_password: ENC[AES256_GCM,data:9YDOz1tiyykz6zSXboWtIg==,iv:j96mRLXGuD4NZcC0Nv1yXFbtOlr6UborqclefZ7J94w=,tag:MqhSewK2NuckTJBf7xu+lA==,type:str]
+miniflux:
+    oauth2_secret: ENC[AES256_GCM,data:/WtZemrdKU8bQbxzrAn437uqoJSO+yZSUDCnxovXV0HFZYQvAn2rbMrgOdoc1OuP,iv:xENKuGDYS1ctnO/WkGv1TxjtQYZ8p64Ik/lMIIam4Q0=,tag:RbAA0LUmsAQDqEM5+a2quQ==,type:str]
+hedgedoc:
+    client_secret: ENC[AES256_GCM,data:J6lRBM7V6F+gPYVyEbOzsFUQe8+3ggP0r58c655DNt7TGgKGdq95pRvLaghMmBCc,iv:i+eLYwnmG1/bKtad2iM2pwEAC3GZLNaKS5ldbubRvyY=,tag:yxaug6YdYo8RR3YOyHd/iA==,type:str]
+forgejo:
+    client_secret: ENC[AES256_GCM,data:5OXhaGzBCbge2tvTaU4ry6/KoavQeYJ45EuakCQJlxb5gMXjRK/s+feF25YJSr2f,iv:TT8j+ciKeSQCZzu1E7D70hWNFpn0cGiomz7jURXjavc=,tag:JVJR033Pc2vaLudaovkl8w==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -24,8 +35,8 @@ sops:
            RzBMVDNjS29SUkdRK3dIV01sU0hYR3cK1SbvKAM6Gpsffv3HIi/WtWnCZUBic0AT
            ZRv4pvJBx1oxWsKIHW0t6VrqWMQ+suup8p6dW+h5HE8Z4ciIMrXLEg==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-02T05:10:32Z"
-    mac: ENC[AES256_GCM,data:ZAdFsjVuk1Fiv+DKmHrc1yu1XQpRDmRHaQhu5hduSZUa1W1cXdTlChvIW5vADFg5tVCjuYptuLvCMW+ZSQeqqG2ntHHZ+IkuovZzKFuc+BIiL/jF2ZzbyJ7X4Wj1GziCScHVxx98dgbpFoufHe6N3wCaHmngo1RYsY5N1RRbRdU=,iv:5IMQ0kOX9UAOm8bcsQRyu6zu8GJjvnHFufCNjY0s9UI=,tag:zBEPSR9DZDpwbCaIka8mXA==,type:str]
+    lastmodified: "2024-12-04T05:07:32Z"
+    mac: ENC[AES256_GCM,data:hD7645epMVYHU6K1AZsHu+fp/PMIqqiZpv7K4Vxzo84slzn0CfZSYaVaYxKNGjOIgEGN9D2FFmq9WL6ChMskMfqqafY7qDpSQqFp9TUwb5jN34XcQg9vplfNw+lMqsnDCt1HENWErRnlDxTI2ctSEcx3UKGBOQ3ttLzUIySdnFY=,iv:reOsqvc8E3l8yxb5gVcqF/rU2o2yKmaUyGNRNT+Skx8=,tag:eBoV8G+X0cPs3Q1xAuv55w==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/biotite/services/forgejo.nix
+++ b/machines/biotite/services/forgejo.nix
@ -0,0 +1,115 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) getExe;
+  inherit (config.my-lib.settings) idpUrl forgejoDomain forgejoGitDomain;
+  settings = {
+    service.DISABLE_REGISTRATION = true;
+    server = {
+      DOMAIN = forgejoDomain;
+      ROOT_URL = "https://${forgejoDomain}";
+      HTTP_ADDR = "/var/run/forgejo/forgejo.sock";
+      START_SSH_SERVER = false;
+      SSH_USER = config.services.forgejo.user;
+      SSH_DOMAIN = forgejoGitDomain;
+      SSH_PORT = 22;
+      PROTOCOL = "http+unix";
+      LFS_MAX_FILE_SIZE = 10737418240;
+      LANDING_PAGE = "/explore/repos";
+    };
+    repository = {
+      ENABLE_PUSH_CREATE_USER = true;
+    };
+    service = {
+      ENABLE_BASIC_AUTHENTICATION = false;
+    };
+    oauth2 = {
+      ENABLED = false; # Disable forgejo as oauth2 provider
+    };
+    oauth2_client = {
+      ACCOUNT_LINKING = "auto";
+      USERNAME = "email";
+      ENABLE_AUTO_REGISTRATION = true;
+      UPDATE_AVATAR = false;
+      OPENID_CONNECT_SCOPES = "openid profile email groups";
+    };
+    metrics = {
+      # ENABLED = true;
+    };
+    other = {
+      SHOW_FOOTER_VERSION = false;
+    };
+  };
+in
+{
+  sops.secrets."forgejo/client_secret" = { };
+  sops.templates."forgejo/env" = {
+    content = ''
+      CLIENT_SECRET=${config.sops.placeholder."forgejo/client_secret"}
+    '';
+    owner = config.systemd.services.forgejo.serviceConfig.User;
+  };
+
+  services.forgejo = {
+    enable = true;
+    inherit settings;
+    # Use cutting edge instead of lts
+    package = pkgs.forgejo;
+    # repositoryRoot = "/mnt/storage/forgejo/repositories";
+    lfs = {
+      enable = true;
+      # contentDir = "/mnt/storage/forgejo/lfs";
+    };
+  };
+
+  systemd.services.forgejo = {
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."forgejo/env".path;
+      preStart =
+        let
+          providerName = "kanidm";
+          args = lib.concatStringsSep " " [
+            "--name ${providerName}"
+            "--provider openidConnect"
+            "--key forgejo"
+            "--secret $CLIENT_SECRET"
+            "--icon-url ${idpUrl}/pkg/img/favicon.png"
+            "--group-claim-name forgejo_role --admin-group Admin"
+          ];
+          exe = getExe config.services.forgejo.package;
+        in
+        ''
+          provider_id=$(${exe} admin auth list | ${pkgs.gnugrep}/bin/grep -w '${providerName}' | cut -f1)
+          if [[ -z "$provider_id" ]]; then
+            ${exe} admin auth add-oauth ${args}
+          else
+            ${exe} admin auth update-oauth --id "$provider_id" ${args}
+          fi
+        '';
+    };
+  };
+
+  users.users.git = {
+    isSystemUser = true;
+    useDefaultShell = true;
+    group = "git";
+    extraGroups = [ "forgejo" ];
+  };
+  users.groups.git = { };
+
+  services.caddy = {
+    virtualHosts."https://${forgejoDomain}".extraConfig = with settings; ''
+      ${
+        if server.PROTOCOL == "http+unix" then
+          "reverse_proxy unix/${server.HTTP_ADDR}"
+        else
+          "reverse_proxy http://${server.HTTP_ADDR}:${toString server.HTTP_PORT}"
+      }
+    '';
+  };
+  users.users.caddy.extraGroups = lib.optional (settings.server.PROTOCOL == "http+unix") "forgejo";
+}
--- a/machines/biotite/services/gotosocial.nix
+++ b/machines/biotite/services/gotosocial.nix
@ -1,4 +1,7 @@
 { config, ... }:
+let
+  inherit (config.my-lib.settings) idpUrl;
+in
 {
  sops.secrets."gotosocial/oidc_client_secret" = {
    owner = "gotosocial";
@ -23,17 +26,17 @@
      instance-expose-public-timeline = true;
      oidc-enabled = true;
      oidc-idp-name = "Kanidm";
-      oidc-issuer = "https://auth.xinyang.life/oauth2/openid/gotosocial";
+      oidc-issuer = "${idpUrl}/oauth2/openid/gotosocial";
      oidc-client-id = "gotosocial";
      oidc-link-existing = true;
    };
+    setupPostgresqlDB = true;
    environmentFile = config.sops.templates."gotosocial.env".path;
  };

  services.caddy = {
    virtualHosts."https://gts.xiny.li".extraConfig = ''
-      encode zstd gzip
-      reverse_proxy * http://${config.services.gotosocial.settings.bind-address}:${toString config.services.gotosocial.settings.port} {
+      reverse_proxy http://${config.services.gotosocial.settings.bind-address}:${toString config.services.gotosocial.settings.port} {
          flush_interval -1
      }
    '';
--- a/machines/biotite/services/hedgedoc.nix
+++ b/machines/biotite/services/hedgedoc.nix
@ -0,0 +1,45 @@
+{ config, pkgs, ... }:
+let
+  inherit (config.my-lib.settings) hedgedocDomain idpUrl;
+in
+{
+  sops.secrets."hedgedoc/client_secret" = { };
+  sops.templates."hedgedoc/env" = {
+    content = ''
+      CMD_OAUTH2_CLIENT_SECRET=${config.sops.placeholder."hedgedoc/client_secret"}
+    '';
+    owner = config.systemd.services.hedgedoc.serviceConfig.User;
+  };
+  services.hedgedoc = {
+    enable = true;
+    environmentFile = config.sops.templates."hedgedoc/env".path;
+    settings = {
+      domain = hedgedocDomain;
+      protocolUseSSL = true; # use SSL for resources
+      path = "/run/hedgedoc/hedgedoc.sock";
+      email = false;
+      allowEmailRegister = false;
+      oauth2 = {
+        baseURL = "${idpUrl}/oauth2/openid/hedgedoc";
+        authorizationURL = "${idpUrl}/ui/oauth2";
+        tokenURL = "${idpUrl}/oauth2/token";
+        userProfileURL = "${idpUrl}/oauth2/openid/hedgedoc/userinfo";
+        userProfileEmailAttr = "email";
+        userProfileUsernameAttr = "name";
+        userProfileDisplayNameAttr = "preferred_name";
+        scope = "openid email profile";
+        clientID = "hedgedoc";
+      };
+      enableStatsApi = true;
+      allowAnonymous = false;
+      defaultPermission = "private";
+    };
+  };
+  services.caddy = {
+    enable = true;
+    virtualHosts."https://${hedgedocDomain}".extraConfig = ''
+      reverse_proxy unix/${config.services.hedgedoc.settings.path}
+    '';
+  };
+  users.users.caddy.extraGroups = [ "hedgedoc" ];
+}
--- a/machines/biotite/services/miniflux.nix
+++ b/machines/biotite/services/miniflux.nix
@ -0,0 +1,35 @@
+{ config, pkgs, ... }:
+let
+  inherit (config.my-lib.settings) idpUrl minifluxUrl;
+in
+{
+  sops = {
+    secrets."miniflux/oauth2_secret" = { };
+  };
+
+  services.miniflux = {
+    enable = true;
+    config = {
+      LOG_LEVEL = "debug";
+      LISTEN_ADDR = "127.0.0.1:58173";
+      BASE_URL = "https://rss.xiny.li/";
+      OAUTH2_PROVIDER = "oidc";
+      OAUTH2_CLIENT_ID = "miniflux";
+      OAUTH2_CLIENT_SECRET_FILE = "%d/oauth2_secret";
+      OAUTH2_REDIRECT_URL = "${minifluxUrl}/oauth2/oidc/callback";
+      OAUTH2_OIDC_DISCOVERY_ENDPOINT = "${idpUrl}/oauth2/openid/miniflux";
+      OAUTH2_USER_CREATION = 1;
+      CREATE_ADMIN = 0;
+    };
+    createDatabaseLocally = true;
+  };
+
+  systemd.services.miniflux.serviceConfig.LoadCredential = [
+    "oauth2_secret:${config.sops.secrets."miniflux/oauth2_secret".path}"
+  ];
+
+  services.caddy.virtualHosts.${minifluxUrl}.extraConfig = ''
+    reverse_proxy ${config.services.miniflux.config.LISTEN_ADDR}
+  '';
+
+}
--- a/machines/biotite/services/restic.nix
+++ b/machines/biotite/services/restic.nix
@ -0,0 +1,55 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  sqliteBackup = fromPath: toPath: file: ''
+    mkdir -p ${toPath}
+    ${lib.getExe pkgs.sqlite} ${fromPath} ".backup '${toPath}/${file}'"
+  '';
+in
+{
+  sops.secrets = {
+    "restic/repo_url" = {
+      sopsFile = ../secrets.yaml;
+    };
+    "restic/repo_password" = {
+      sopsFile = ../secrets.yaml;
+    };
+  };
+
+  custom.restic = {
+    enable = true;
+    paths = [
+      "/backup/db"
+      "/backup/var/lib"
+    ];
+    backupPrepareCommand = [
+      ''
+        mkdir -p /backup/var
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r /var/lib /backup/var/lib
+      ''
+    ];
+    backupCleanupCommand = [
+      ''
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup/var/lib
+      ''
+    ];
+    btrfsRoots = [ ];
+  };
+
+  services.postgresqlBackup = {
+    enable = true;
+    compression = "zstd";
+    compressionLevel = 9;
+    location = "/backup/db/postgresql";
+  };
+
+  services.restic.backups.${config.networking.hostName} = {
+    extraBackupArgs = [
+      "--limit-upload=1024"
+    ];
+  };
+}
--- a/machines/biotite/services/synapse.nix
+++ b/machines/biotite/services/synapse.nix
@ -0,0 +1,120 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (config.my-lib.settings) idpUrl synapseDelegateUrl synapseUrl;
+  port-synapse = 6823;
+in
+{
+  sops.secrets."synapse/oidc_client_secret" = {
+    owner = "matrix-synapse";
+  };
+
+  nixpkgs.config.permittedInsecurePackages = [
+    "olm-3.2.16"
+  ];
+
+  services.postgresql = {
+    # Not using ensure here because LC_COLLATE and LC_CTYPE must be provided
+    # at db creation
+    initialScript = pkgs.writeText "synapse-init.sql" ''
+      CREATE ROLE "matrix-synapse" WITH LOGIN PASSWORD 'synapse';
+      CREATE DATABASE "matrix-synapse" WITH OWNER "matrix-synapse"
+        TEMPLATE template0
+        LC_COLLATE = "C"
+        LC_CTYPE = "C";
+    '';
+  };
+
+  services.matrix-synapse = {
+    enable = true;
+    withJemalloc = true;
+    settings = {
+      server_name = "xiny.li";
+      public_baseurl = synapseDelegateUrl;
+      database = {
+        name = "psycopg2";
+        args = {
+          user = "matrix-synapse";
+        };
+      };
+      listeners = [
+        {
+          bind_addresses = [
+            "127.0.0.1"
+          ];
+          port = port-synapse;
+          resources = [
+            {
+              compress = true;
+              names = [
+                "client"
+                "federation"
+              ];
+            }
+          ];
+          tls = false;
+          type = "http";
+          x_forwarded = true;
+        }
+      ];
+      experimental_features = {
+        # Room summary api
+        msc3266_enabled = true;
+        # Removing account data
+        msc3391_enabled = true;
+        # Thread notifications
+        msc3773_enabled = true;
+        # Remotely toggle push notifications for another client
+        msc3881_enabled = true;
+        # Remotely silence local notifications
+        msc3890_enabled = true;
+        # Remove legacy mentions
+        msc4210_enabled = true;
+      };
+      oidc_providers = [
+        {
+          idp_id = "Kanidm";
+          idp_name = lib.removePrefix "https://" idpUrl;
+          issuer = "${idpUrl}/oauth2/openid/synapse";
+          authorization_endpoint = "${idpUrl}/ui/oauth2";
+          token_endpoint = "${idpUrl}/oauth2/token";
+          userinfo_endpoint = "${idpUrl}/oauth2/openid/synapse/userinfo";
+          client_id = "synapse";
+          client_secret_path = config.sops.secrets."synapse/oidc_client_secret".path;
+          scopes = [
+            "openid"
+            "profile"
+          ];
+          allow_existing_users = true;
+          backchannel_logout_enabled = true;
+          user_mapping_provider.config = {
+            confirm_localpart = true;
+            localpart_template = "{{ user.preferred_username }}";
+            display_name_template = "{{ user.name }}";
+          };
+        }
+      ];
+    };
+  };
+
+  services.caddy = {
+    virtualHosts.${synapseUrl}.extraConfig = ''
+      header /.well-known/matrix/* Content-Type application/json
+      header /.well-known/matrix/* Access-Control-Allow-Origin *
+      respond /.well-known/matrix/server `{"m.server":"synapse.xiny.li:443"}`
+      respond /.well-known/matrix/client `{"m.homeserver":{"base_url":"${synapseDelegateUrl}"}}`
+    '';
+    virtualHosts.${synapseDelegateUrl}.extraConfig = ''
+      reverse_proxy /_matrix/* 127.0.0.1:${toString port-synapse}
+      reverse_proxy /_synapse/client/* 127.0.0.1:${toString port-synapse}
+    '';
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    443
+  ];
+}
--- a/machines/biotite/services/vaultwarden.nix
+++ b/machines/biotite/services/vaultwarden.nix
@ -0,0 +1,25 @@
+{ config, pkgs, ... }:
+let
+  inherit (config.my-lib.settings) vaultwardenUrl;
+in
+{
+  services.vaultwarden = {
+    enable = true;
+    dbBackend = "sqlite";
+    config = {
+      DOMAIN = "${vaultwardenUrl}";
+      SIGNUPS_ALLOWED = false;
+
+      ROCKET_ADDRESS = "127.0.0.1";
+      ROCKET_PORT = 8222;
+
+      ROCKET_LOG = "normal";
+    };
+  };
+
+  services.caddy = {
+    virtualHosts.${vaultwardenUrl}.extraConfig = with config.services.vaultwarden.config; ''
+      reverse_proxy ${ROCKET_ADDRESS}:${toString ROCKET_PORT}
+    '';
+  };
+}
--- a/machines/calcite/configuration.nix
+++ b/machines/calcite/configuration.nix
@ -51,7 +51,6 @@ in
  };
  # services.gnome.gnome-keyring.enable = lib.mkForce false;
  security.pam.services.login.enableGnomeKeyring = lib.mkForce false;
-  services.ssh-tpm-agent.enable = true;

  programs.ssh.agentPKCS11Whitelist = "${config.security.tpm2.pkcs11.package}/lib/libtpm_pkcs11.so";

@ -66,18 +65,7 @@ in
    };
  };

-  programs.oidc-agent.enable = true;
-  programs.oidc-agent.providers = [
-    {
-      issuer = "https://home.xinyang.life:9201";
-      pubclient = {
-        client_id = "xdXOt13JKxym1B1QcEncf2XDkLAexMBFwiT9j6EfhhHFJhs2KM9jbjTmf8JBXE69";
-        client_secret = "UBntmLjC2yYCeHwsyj73Uwo9TAaecAetRwMw0xYcvNL9yRdLSUi0hUAHfvCHFeFh";
-        scope = "openid offline_access profile email";
-      };
-    }
-  ];
-
+  programs.vim.enable = true;
  programs.vim.defaultEditor = true;

  # Keep this even if enabled in home manager
@ -307,13 +295,7 @@ in
    bitwarden

    # Browser
-    (chromium.override {
-      commandLineArgs = [
-        "--ozone-platform-hint=auto"
-        "--enable-wayland-ime"
-      ];
-    })
-    brave
+    chromium

    # Writting
    zotero
@ -352,16 +334,16 @@ in
    ];
  };

-  custom.forgejo-actions-runner = {
-    enable = false;
-    tokenFile = config.sops.secrets."gitea/envfile".path;
-    settings = {
-      runner.capacity = 2;
-      runner.fetch_timeout = "120s";
-      runner.fetch_interval = "30s";
-    };
-  };
-
+  # custom.forgejo-actions-runner = {
+  #   enable = false;
+  #   tokenFile = config.sops.secrets."gitea/envfile".path;
+  #   settings = {
+  #     runner.capacity = 2;
+  #     runner.fetch_timeout = "120s";
+  #     runner.fetch_interval = "30s";
+  #   };
+  # };
+  #
  custom.prometheus = {
    exporters.node.enable = true;
  };
@ -379,15 +361,12 @@ in
  # Fonts
  fonts = {
    packages = with pkgs; [
-      (nerdfonts.override {
-        fonts = [
-          "FiraCode"
-          "FiraMono"
-          "JetBrainsMono"
-          "RobotoMono"
-          "Ubuntu"
-        ];
-      })
+      nerd-fonts.ubuntu-sans
+      nerd-fonts.ubuntu
+      nerd-fonts.fira-code
+      nerd-fonts.fira-mono
+      nerd-fonts.jetbrains-mono
+      nerd-fonts.roboto-mono
      noto-fonts
      noto-fonts-emoji
      liberation_ttf
--- a/machines/calcite/network.nix
+++ b/machines/calcite/network.nix
@ -1,4 +1,9 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:

 {
  imports = [ ];
@ -24,16 +29,9 @@

  services.dae.enable = true;
  services.dae.configFile = "/var/lib/dae/config.dae";
-  systemd.services.dae.after = lib.mkIf (config.networking.networkmanager.enable) [ "NetworkManager-wait-online.service" ];
-
-  custom.sing-box = {
-    enable = false;
-    configFile = {
-      urlFile = config.sops.secrets.sing_box_url.path;
-      hash = "6ca5bc8a16f8c413227690aceeee2c12c02cab09473c216b849af1e854b98588";
-    };
-    overrideSettings.experimental.clash_api.external_ui = "${config.nur.repos.linyinfeng.yacd}";
-  };
+  systemd.services.dae.after = lib.mkIf (config.networking.networkmanager.enable) [
+    "NetworkManager-wait-online.service"
+  ];

  # Open ports in the firewall.
  networking.firewall.enable = true;
--- a/machines/massicot/default.nix
+++ b/machines/massicot/default.nix
@ -15,24 +15,9 @@
    defaultSopsFile = ./secrets.yaml;
    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
    secrets = {
-      storage_box_mount = {
-        owner = "root";
-      };
      gts_env = {
        owner = "gotosocial";
      };
-      hedgedoc_env = {
-        owner = "hedgedoc";
-      };
-      grafana_oauth_secret = {
-        owner = "grafana";
-      };
-      "miniflux/oauth2_secret" = {
-        owner = "root";
-      };
-      "forgejo/env" = {
-        owner = "forgejo";
-      };
    };
  };

--- a/machines/massicot/kanidm-provision.nix
+++ b/machines/massicot/kanidm-provision.nix
@ -1,4 +1,14 @@
-{ config, lib, ... }:
+{ pkgs, config, ... }:
+let
+  inherit (config.my-lib.settings)
+    gotosocialUrl
+    minifluxUrl
+    hedgedocDomain
+    forgejoDomain
+    grafanaUrl
+    synapseDelegateUrl
+    ;
+in
 {
  services.kanidm.provision = {
    enable = true;
@ -45,6 +55,9 @@
      miniflux-users = {
        members = [ "xin" ];
      };
+      synapse-users = {
+        members = [ "xin" ];
+      };
      idm_people_self_mail_write = {
        members = [ ];
      };
@ -73,8 +86,8 @@
    systems.oauth2 = {
      forgejo = {
        displayName = "ForgeJo";
-        originUrl = "https://git.xinyang.life/user/oauth2/kanidm/callback";
-        originLanding = "https://git.xinyang.life/user/oauth2/kanidm";
+        originUrl = "https://${forgejoDomain}/user/oauth2/kanidm/callback";
+        originLanding = "https://${forgejoDomain}/user/oauth2/kanidm";
        allowInsecureClientDisablePkce = true;
        scopeMaps = {
          forgejo-access = [
@ -110,8 +123,8 @@
      };
      gotosocial = {
        displayName = "GoToSocial";
-        originUrl = "https://gts.xiny.li/auth/callback";
-        originLanding = "https://gts.xiny.li/auth/callback";
+        originUrl = "${gotosocialUrl}/auth/callback";
+        originLanding = "${gotosocialUrl}/auth/callback";
        allowInsecureClientDisablePkce = true;
        scopeMaps = {
          gts-users = [
@ -147,8 +160,8 @@

      hedgedoc = {
        displayName = "HedgeDoc";
-        originUrl = "https://docs.xinyang.life/auth/oauth2/callback";
-        originLanding = "https://docs.xinyang.life/auth/oauth2";
+        originUrl = "https://${hedgedocDomain}/auth/oauth2/callback";
+        originLanding = "https://${hedgedocDomain}/auth/oauth2";
        allowInsecureClientDisablePkce = true;
        scopeMaps = {
          hedgedoc-users = [
@ -177,9 +190,8 @@
      };
      miniflux = {
        displayName = "Miniflux";
-        originUrl = "https://rss.xinyang.life/oauth2/oidc/callback";
-
-        originLanding = "https://rss.xinyang.life/oauth2/oidc/redirect";
+        originUrl = "${minifluxUrl}/oauth2/oidc/callback";
+        originLanding = "${minifluxUrl}/oauth2/oidc/redirect";
        scopeMaps = {
          miniflux-users = [
            "openid"
@ -190,8 +202,8 @@
      };
      grafana = {
        displayName = "Grafana";
-        originUrl = "https://grafana.xinyang.life/login/generic_oauth";
-        originLanding = "https://grafana.xinyang.life/";
+        originUrl = "${grafanaUrl}/login/generic_oauth";
+        originLanding = "${grafanaUrl}/";
        scopeMaps = {
          grafana-users = [
            "openid"
@ -211,6 +223,17 @@
          };
        };
      };
+      synapse = {
+        displayName = "Synapse";
+        originUrl = "${synapseDelegateUrl}/_synapse/client/oidc/callback";
+        originLanding = "${synapseDelegateUrl}/";
+        scopeMaps = {
+          synapse-users = [
+            "openid"
+            "profile"
+          ];
+        };
+      };
    };
  };
 }
--- a/machines/massicot/services.nix
+++ b/machines/massicot/services.nix
@ -23,26 +23,6 @@ in
    8448
  ];

-  custom.vaultwarden = {
-    enable = true;
-    domain = "vaultwarden.xinyang.life";
-  };
-
-  custom.hedgedoc = {
-    enable = true;
-    caddy = true;
-    domain = "docs.xinyang.life";
-    mediaPath = "/mnt/storage/hedgedoc";
-    oidc = {
-      enable = true;
-      baseURL = "https://auth.xinyang.life/oauth2/openid/hedgedoc";
-      authorizationURL = "https://auth.xinyang.life/ui/oauth2";
-      tokenURL = "https://auth.xinyang.life/oauth2/token";
-      userProfileURL = "https://auth.xinyang.life/oauth2/openid/hedgedoc/userinfo";
-    };
-    environmentFile = config.sops.secrets.hedgedoc_env.path;
-  };
-
  custom.monitoring = {
    promtail.enable = true;
  };
@ -92,40 +72,6 @@ in
    };
  };

-  custom.miniflux = {
-    enable = true;
-    environment = {
-      LOG_LEVEL = "debug";
-      LISTEN_ADDR = "127.0.0.1:58173";
-      BASE_URL = "https://rss.xinyang.life/";
-      OAUTH2_PROVIDER = "oidc";
-      OAUTH2_CLIENT_ID = "miniflux";
-      OAUTH2_REDIRECT_URL = "https://rss.xinyang.life/oauth2/oidc/callback";
-      OAUTH2_OIDC_DISCOVERY_ENDPOINT = "https://auth.xinyang.life/oauth2/openid/miniflux";
-      OAUTH2_USER_CREATION = 1;
-    };
-    oauth2SecretFile = config.sops.secrets."miniflux/oauth2_secret".path;
-  };
-
-  services.matrix-conduit = {
-    enable = true;
-    package = pkgs.matrix-conduit;
-    settings.global = {
-      server_name = "xinyang.life";
-      port = 6167;
-      # database_path = "/var/lib/matrix-conduit/";
-      max_concurrent_requests = 100;
-      log = "info";
-      database_backend = "rocksdb";
-      allow_registration = false;
-
-      well_known = {
-        client = "https://msg.xinyang.life";
-        server = "msg.xinyang.life:443";
-      };
-    };
-  };
-
  users.users.conduit = {
    isSystemUser = true;
    group = "conduit";
@ -150,111 +96,6 @@ in
    environmentFile = config.sops.secrets.gts_env.path;
  };

-  services.forgejo = {
-    enable = true;
-    # Use cutting edge instead of lts
-    package = pkgs.forgejo;
-    repositoryRoot = "/mnt/storage/forgejo/repositories";
-    lfs = {
-      enable = true;
-      contentDir = "/mnt/storage/forgejo/lfs";
-    };
-    settings = {
-      service.DISABLE_REGISTRATION = true;
-      server = {
-        ROOT_URL = "https://git.xinyang.life/";
-        START_SSH_SERVER = false;
-        SSH_USER = config.services.forgejo.user;
-        SSH_DOMAIN = "ssh.xinyang.life";
-        SSH_PORT = 22;
-        LFS_MAX_FILE_SIZE = 10737418240;
-        LANDING_PAGE = "/explore/repos";
-      };
-      repository = {
-        ENABLE_PUSH_CREATE_USER = true;
-      };
-      service = {
-        ENABLE_BASIC_AUTHENTICATION = false;
-      };
-      oauth2 = {
-        ENABLED = false; # Disable forgejo as oauth2 provider
-      };
-      oauth2_client = {
-        ACCOUNT_LINKING = "auto";
-        USERNAME = "email";
-        ENABLE_AUTO_REGISTRATION = true;
-        UPDATE_AVATAR = false;
-        OPENID_CONNECT_SCOPES = "openid profile email groups";
-      };
-      other = {
-        SHOW_FOOTER_VERSION = false;
-      };
-    };
-  };
-
-  systemd.services.forgejo = {
-    serviceConfig = {
-      EnvironmentFile = config.sops.secrets."forgejo/env".path;
-      ExecStartPost = ''
-        ${lib.getExe config.services.forgejo.package} admin auth update-oauth \
-            --id 1 \
-            --name kanidm \
-            --provider openidConnect \
-            --key forgejo \
-            --secret $CLIENT_SECRET \
-            --icon-url https://auth.xinyang.life/pkg/img/favicon.png \
-            --group-claim-name forgejo_role --admin-group Admin
-      '';
-    };
-  };
-
-  services.grafana = {
-    enable = true;
-    settings = {
-      server = {
-        http_addr = "127.0.0.1";
-        http_port = 3003;
-        root_url = "https://grafana.xinyang.life";
-        domain = "grafana.xinyang.life";
-      };
-      "auth.generic_oauth" = {
-        enabled = true;
-        name = "Kanidm";
-        client_id = "grafana";
-        scopes = "openid,profile,email,groups";
-        auth_url = "https://auth.xinyang.life/ui/oauth2";
-        token_url = "https://auth.xinyang.life/oauth2/token";
-        api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
-        use_pkce = true;
-        use_refresh_token = true;
-        allow_sign_up = true;
-        login_attribute_path = "preferred_username";
-        groups_attribute_path = "groups";
-        role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
-        allow_assign_grafana_admin = true;
-        auto_login = true;
-      };
-      "auth" = {
-        disable_login_form = true;
-      };
-    };
-  };
-
-  systemd.services.grafana.serviceConfig.EnvironmentFile =
-    config.sops.secrets.grafana_oauth_secret.path;
-
-  users.users.git = {
-    isSystemUser = true;
-    useDefaultShell = true;
-    group = "git";
-    extraGroups = [ "forgejo" ];
-  };
-  users.groups.git = { };
-
-  users.users = {
-    ${config.services.caddy.user}.extraGroups = [ config.services.ntfy-sh.group ];
-  };
-
  services.caddy = {
    enable = true;
    virtualHosts."xinyang.life:443".extraConfig = ''
@ -265,12 +106,6 @@ in
          flush_interval -1
      }
    '';
-    virtualHosts."https://msg.xinyang.life:443".extraConfig = ''
-      reverse_proxy /_matrix/* localhost:6167
-    '';
-    virtualHosts."https://git.xinyang.life:443".extraConfig = ''
-      reverse_proxy http://${config.services.gitea.settings.server.DOMAIN}:${toString config.services.gitea.settings.server.HTTP_PORT}
-    '';

    virtualHosts."http://auth.xinyang.life:80".extraConfig = ''
      reverse_proxy ${config.security.acme.certs."auth.xinyang.life".listenHTTP}
@ -285,10 +120,6 @@ in
      }
    '';

-    virtualHosts."https://rss.xinyang.life".extraConfig = ''
-      reverse_proxy ${config.custom.miniflux.environment.LISTEN_ADDR}
-    '';
-
    virtualHosts."https://ntfy.xinyang.life".extraConfig = ''
      reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix}
      @httpget {
@ -298,13 +129,5 @@ in
      }
      redir @httpget https://{host}{uri}
    '';
-
-    virtualHosts."https://grafana.xinyang.life".extraConfig =
-      let
-        grafanaSettings = config.services.grafana.settings.server;
-      in
-      ''
-        reverse_proxy http://${grafanaSettings.http_addr}:${toString grafanaSettings.http_port}
-      '';
  };
 }
--- a/machines/massicot/services/restic.nix
+++ b/machines/massicot/services/restic.nix
@ -34,13 +34,6 @@ in
    ];
  };

-  services.postgresqlBackup = {
-    enable = true;
-    compression = "zstd";
-    compressionLevel = 9;
-    location = "/backup/postgresql";
-  };
-
  services.restic.backups.${config.networking.hostName} = {
    extraBackupArgs = [
      "--limit-upload=1024"
--- a/machines/secrets.yaml
+++ b/machines/secrets.yaml
@ -10,74 +10,83 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwMHB1bFQ3dWJIU3NiOVVP
-            Yi9LZE1PTVdMY1BqS1JHV3VPLzZIY0hGK0NZClNlclVXKzBvNTBrTlhiR0VsaVoz
-            RlVLNVBEVDgzSXB5ZGxDd3hqNDh2V2MKLS0tIEhBZHFUY3c2VXJBVEVKamZ6TzBa
-            MlFsNnVEV0xCdlJoRnBhUHF2MmswUEUKNYD9zssGBy9SaKeOMvTz71B6KMPW87cM
-            tFJzgnQceEQF658lVa5cCzG1gzraCgBtQU15XzC7e8zWI9CHquRRlQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5SjAzOEozUzh1bzVvaHgr
+            T2xsVUszTHVSdWIyM3B5TFhtUEFMeVZlYzNrCk5IOWFNbTErbTVkQnNlVllMZWlV
+            Q2lHZXRIdzBiRFRSZnNUVWd2NXVXVGcKLS0tIERhcjh3VVlqSGxHUHpnc1JzVksv
+            VXpQVVVCUC9xR3crWm9rTk13LzVhK1EKwiuvwx3ZhcDE+9w7/dR4PrZSSoJMvklT
+            m7I32dMRk0o9zcl5KYU5L9Hwb+z+EBE34raoGKBF5K4aQcbZQUX3Cw==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTTnZLTlZQRzc1enVEa1BN
-            SHdoSi9oOXk4UTV0SlRZS2tLS2FFL3VjNzNNClVWTTNKekF6T0RTUzdEeWhLbHoz
-            WFZKaHJEaVBWa04zRWRiVnJZRjU0YVEKLS0tIFJVL0FEemowS3V6MmsxbWJMU2I1
-            U2NnUnVKdFlRSGVzUFQ4ZFcwL0lWTlkKz1t3yqjgIdMWS/Nsy2nq3oCjOhGDP+UT
-            L+LAuFExJPV0qlsOG/kCGB/WtCJfnBvcp6vPDBLqjK8NllIX/iPI5g==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5R1ZIRlN2b3M2OUQ0T2cw
+            eE5DTm9KY1NUY1p5eDhLNG4xMDVkVjRyWDNRClp3MTRWeGJMYTczcC9YQTNZdkxx
+            ejJ3QnhjcUcyUldUNEVqVUh6Z2grd00KLS0tIDVvbDZWbmZPZVhDNHM1K1kzaE95
+            aHJqSU16dlJiRGl0VWNMVXVYMmhPb2MKMboq9ShGIJMFVENgLPlQdwdtTOjVb0CC
+            4ttM3xWnYkf8416a0OYFrda5l1kfJJzQakbk/tbGcTu1yTcd+6lOtA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVby8wYS9pa0szTlVUS3FI
+            VWhjaCtyUzNLbkw2VXRlWkVMZlRkeXJMZGlRCnBTWklnZ0Uzd2lTMGt1M2wxZ0px
+            NFl2RW5hSUZVdHI0aVFRMHJtMFQ3ODAKLS0tIFlYOHVRYVFGbkcvUWRmQitQQnI5
+            bG5vemMvcWdpOEtxNGRpS0doQmtuUFkK8Hxl//kOtbEw3jf96ZZ4G1Yb94f4Jeb4
+            TfPs7O/ESJY8ovNsoXRQEt99vOR5D1wBzyZBY9E3f2ZzY/uBmup0cw==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETWpkcjhINktqeGxjdWxz
-            UTVVNC9kalorcVJOdHpJSkZJNXlGUHZ2VUdrCjRCclBTZnJEZ3JGOVpqS1Y0b0dt
-            eldFMS91WUc2Y1FnWWZoN0grc01pT0UKLS0tIC96TjlEaVBGRkZhZ0hac2lmbEdI
-            eHMzTFhsQ0FqY05uUEZSbExCcmdscEkKdxITlc0V5ayq+9fmj77SnEMFxKJhOOta
-            RfJhOQUv8g3nCN+SsuaOy0TitUCiDWh5XoB0DufEQPcS/kzGZN1Inw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPSmRYMkNIdERJZVBxV1p1
+            emlqOTBpN3l2WXkzNjRRcFI5NUZDZnQ1WXdnCkRVbm8xais5aGVCTmtSTGxaTXlT
+            L2ZWQ0p5WFZNRWl5SWVkRUYwc2R3b1UKLS0tIEZEck4yMmJUQWVvNHRJQnpCQTBo
+            cDJsaG83MTdXWVd2NUpLczhjWTBBZVUK5BxBIYVqkqVLw9LTbnJ8SQWN2i4USdI8
+            8m/hZFXTJ4GI0f795DEmbcZq9xET14aQqta0wSASqwP/5Ld1mo0a0w==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBydlQ4S1duQU53Wk1nd21K
-            d2RqM1F0VDFJVXB2aGRTZ2hxczI2V1lndVdrCjArVlE2N0RGZ0htUEZYdVlQMlU5
-            SWIwWHVCaWxaQTJMNzg3WC8xRS9IYzgKLS0tIDRvSS8ybVlrSy9zYjQ2NXBaMlZk
-            Ulg4cUFBejRoS3VEWkRaZEUxMExUeWMKNeq6TN1gaBNU9vAitGttcU+8HmFQipdm
-            LPwo4/toyf27emb4KGs0AV0Dm4Sxj9S3Xvrv1B+qvhfT638/RIUm2w==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSkhjRTdBWklZUEpUanM0
+            Wjl4b2c3K0g0ZUxxMlRrUFhhZzhNRXhPVnpvCmpNWVBNTXNYczV3aWhCd05FOGJ0
+            YlNobFhWdStGbDRZV2NlUWV6ZFRVNEkKLS0tIGd1RUR4K21GOEQ0aWtqRi9RREpE
+            RXBXcXFYUDVXVzN4Q25zSklFU21wbFkKQuTHkgFC5HRPO7/PuVhJzbbHOTPaFXvN
+            +Y31AK3OAVdUETMEuJ2mk50Bi5BiiUeOnnv1bZ6O+iX0o20ysUseTg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4YXpyOXE3MFovWEQvMVRr
-            TGVST3U0N2dCVDJGT1A3eUtlRis3bFEvTHlFClZHQ2xRWklMMCtER01QNEVHaVYr
-            MC94V3R4MVdNdUU3eXQ2RGFFVGo4VFEKLS0tIDQ4b2ZuMy9URUswWUZqNHlxandU
-            OFducVVzdGZGY0tnbFFBZDdjVzVkaUEKN8qAbbrd4pAHRGIN8O64fl7bQ6hx6Isr
-            Qx0xKeuhJCVXgtE8xc7xmnEhqrcONlflJ/XUnYV9jOkB71zSBJxruA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnc3NOZFRYT1VnaVZSaTRi
+            WnluSEk4d1U5TWx2REZRZ3VCRVp2ZzlKY0NvCjNlUnIwdWVqSnlQOWp1dlJ5THlW
+            c2xTNHhnaE94a2ZTeXJjQTVxeGRLTmsKLS0tIFV4c2NZK1ZnL2xtUlVvSksxNi9o
+            L3dodkJXVjZrekVldTVsRFRxSFlrTmMKiokjgIRIsI8D2aFP/Qem4iGzC4yr5lm2
+            ZwggC/UfD56ysTEqrVaDnR7f5fSqZLWdstPJn7I/vr5CwKRMbMPYSA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzczdPMDdWU1ZtckJRQm5j
-            UWJub0Yzd3NzOEh4YWdId01nYWI1YVY3dng0ClpEYXBJV2cvWEdjdXcwUFI3Y0NG
-            MDgvTmNZOXRQQndyVmRHamNRbzVaVU0KLS0tIGFKVTI4TkE2UjhDUSsxQTlNQ0Vk
-            QmFMNnlqbnhScC90T012K1QxRnRUOHcKAV7NxUn0CMcjKwK8zrocoLO1P9jc22uG
-            eG+vdJ6xzA99UX51aPxQOeEJgdFPEd3y1QJszQmRzThvid7y4lv0Cw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpN0llOTBJU1pNNVFxVWxt
+            aFdKdStKL1ZlZ0p6WFRQbHpGNnpmdlJXdG1FCkx5eDhZWWJvQ2xSWEJqWnZ6NmNt
+            Y0MzNDg5QzVSbEZteW1LNlFyRFg5Q0EKLS0tIDBrT0dEZlBoTExYcGRNZjZ5Znpz
+            cnE4YWRTMmRsTENhOTl5R2dYSzQwazAKvnTvZz842Mg5AVlIoYHI2BG+0/hO5zIv
+            jRVJri98fgGterXADTPmeoY3p+fFQggTPhs/5s5GSQxd5aiX8vvvrA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsVmpzenRvWE5EK2wzRFkx
-            SERZV0s1Rkt0ZnZ1U3JQSFNhdGVvaWhWcTA4CjVxK0Z0MHI0ZnMrUS9YYWhTTG1z
-            L2lVS1Q2UkVQd2x5b1E1eWpQVGp2ZHMKLS0tIHNLOGhTYjkzWkFEM05wYkRZeXFQ
-            SXNTSGZZSFE2bFhybXdIc1FUb1ZBd0kKkYzflPRk6GrE6t9oVGOzc8xcyZDxiIw8
-            9SVXIgV0WVpY4lnFKYKH2i4+1sIm6tKOpizlQxTg5VgmmrTtfazWAA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPQWljdGg4VTlDdGhoblpk
+            LytxK2FnQVI1dzB2bnFaWUtoUVNGS3lpU3prCnRwUTNnZVVXTnZ6eCtScTk5YzI3
+            TGM2MmNhaHQ3NXAzMk0rcnJoTlp5STQKLS0tIEp2U3YvUUhXTkt3VFczY3J1LzMv
+            ZzM0VHpqamRIZVROS2lQdXFhQTNBekEKEySldC+VvZvPY398ZVkB5s73bT3QbuLh
+            IqTv+wbkbjlvZJUavVyycY5SwMXkSX3ge9W/64mt/RDs88gSXFS+Sw==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0NHpkOTFHaXRhVGNua0dV
-            alRieWJ6WG5ZNzlvcTR2aTVUeWFBVGVVUUNZCnY2VUZUOWVlNGY1ZldyVGE2bkpi
-            VXVtQ3IyK0kyV1cyMU5nN1lYaW1oOUkKLS0tIFRVRGFCNWlGendSVEhHY0w0QTl6
-            emJEQkQ3QlU0TFVWaW1uQytaUndmQlEKKahqJpX8vI+PASOzzod/sFvXSkQFnJ9O
-            YmnmiFxm5WZDPLHwkgVx8FgCq9RfAad4HybhsMjYPKXJ/fNa/WVZRA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4bGppem15NlVod2hCRkM5
+            MzY1aUZOdEVzRzdEYTRNakdMQWJlRkk0eEZzClRLSnRrQUoreU5MVG40KzRKSGcw
+            bUU4ZnpLU0VtOWxXVllrSW5lN0NWb0kKLS0tIE1iemRlVVpieEhxRnlIb2dFUHZr
+            am04NVRtU2N6SThYZWdXVE5RZ1B2aE0KVcHvB5k2Gcu/St0P8WPFzlCtuZthZTKo
+            hwVc0lC6Xxt25hriaUFinwnyvcjxrLCx0Nq7f9Zn16nJcza5kev1nQ==
            -----END AGE ENCRYPTED FILE-----
    lastmodified: "2024-11-30T06:31:42Z"
    mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str]
--- a/machines/thorite/default.nix
+++ b/machines/thorite/default.nix
@ -1,7 +1,10 @@
+{ config, ... }:
 {
  imports = [
    ./hardware-configurations.nix
    ./monitoring.nix
+    ./restic.nix
+    ./ntfy.nix
  ];

  config = {
@ -28,6 +31,10 @@
      443
    ];

+    services.tailscale.enable = true;
+
+    services.caddy.enable = true;
+
    commonSettings = {
      auth.enable = true;
    };
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@ -1,5 +1,20 @@
-{ config, my-lib, ... }:
-with my-lib;
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with config.my-lib;
+let
+  inherit (config.my-lib.settings)
+    minifluxUrl
+    gotosocialUrl
+    hedgedocDomain
+    grafanaUrl
+    ntfyUrl
+    ;
+  removeHttps = s: lib.removePrefix "https://" s;
+in
 {
  config = {
    sops = {
@ -14,11 +29,23 @@ with my-lib;

    custom.monitoring = {
      grafana.enable = true;
-      loki.enable = true;
+      loki = {
+        enable = true;
+        rules = {
+          sshd_closed = {
+            expr = ''count_over_time({unit="sshd.service"} |~ "Connection closed by authenticating user" [15m]) > 25'';
+            description = "More then 25 login attemps in last 15 min without success";
+          };
+          unusual_log_volume = {
+            expr = ''sum by (unit) (rate({unit=~".+"}[5m])) > 80'';
+            description = "Unit {{ $labels.unit }} is logging at an unusually high rate";
+          };
+        };
+      };
      promtail.enable = true;
    };

-    services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =
+    services.caddy.virtualHosts.${grafanaUrl}.extraConfig =
      with config.services.grafana.settings.server; ''
        reverse_proxy http://${http_addr}:${toString http_port}
      '';
@ -30,7 +57,10 @@ with my-lib;
        blackbox.enable = true;
        node.enable = true;
      };
-      ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]);
+      ruleModules =
+        (mkCaddyRules [ { host = "thorite"; } ])
+        ++ (mkNodeRules [ { host = "thorite"; } ])
+        ++ (mkBlackboxRules [ { host = "thorite"; } ]);
    };

    services.prometheus.scrapeConfigs =
@ -39,8 +69,6 @@ with my-lib;
          "la-00.video.namely.icu:8080"
          "fre-00.video.namely.icu:8080"
          "hk-00.video.namely.icu:8080"
-          "49.13.13.122:443"
-          "45.142.178.32:22"
          "home.xinyang.life:8000"
        ];
        passwordFile = config.sops.secrets."prometheus/metrics_password".path;
@ -52,27 +80,44 @@ with my-lib;
          address = "weilite.coho-tet.ts.net";
          port = 8082;
        }
+        {
+          name = "restic_rest_server";
+          address = "backup.xinyang.life";
+          port = 8443;
+        }
        {
          inherit passwordFile;
          name = "gotosocial";
-          address = "xinyang.life";
+          address = removeHttps gotosocialUrl;
        }
        {
          inherit passwordFile;
          name = "miniflux";
-          address = "rss.xinyang.life";
+          address = removeHttps minifluxUrl;
+        }
+        {
+          name = "hedgedoc";
+          address = hedgedocDomain;
        }
        {
          name = "ntfy";
-          address = "ntfy.xinyang.life";
+          address = removeHttps ntfyUrl;
        }
        {
          name = "grafana-eu";
-          address = "grafana.xinyang.life";
+          address = removeHttps grafanaUrl;
+        }
+        {
+          name = "loki";
+          scheme = "http";
+          address = "thorite.coho-tet.ts.net";
+          port = 3100;
        }
      ])
      ++ (mkCaddyScrapes [
        { address = "thorite.coho-tet.ts.net"; }
+        { address = "biotite.coho-tet.ts.net"; }
+        { address = "weilite.coho-tet.ts.net"; }
      ])
      ++ (mkNodeScrapes [
        { address = "thorite.coho-tet.ts.net"; }
@ -85,11 +130,11 @@ with my-lib;
      ++ (mkBlackboxScrapes [
        {
          hostAddress = "thorite.coho-tet.ts.net";
-          targetAddresses = probeList;
+          targetAddresses = probeList ++ [ "49.13.13.122:443" ];
        }
        {
          hostAddress = "massicot.coho-tet.ts.net";
-          targetAddresses = probeList;
+          targetAddresses = probeList ++ [ "45.142.178.32:443" ];
        }
        {
          hostAddress = "weilite.coho-tet.ts.net";
--- a/machines/thorite/ntfy.nix
+++ b/machines/thorite/ntfy.nix
@ -0,0 +1,29 @@
+{ config, ... }:
+let
+  inherit (config.my-lib.settings) ntfyUrl;
+in
+{
+
+  services.ntfy-sh = {
+    enable = true;
+    group = "caddy";
+    settings = {
+      listen-unix = "/var/run/ntfy-sh/ntfy.sock";
+      listen-unix-mode = 432; # octal 0660
+      base-url = ntfyUrl;
+    };
+  };
+
+  systemd.services.ntfy-sh.serviceConfig.RuntimeDirectory = "ntfy-sh";
+
+  services.caddy.virtualHosts.${ntfyUrl}.extraConfig = ''
+    reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix}
+    @httpget {
+      protocol http
+      method GET
+      path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/)
+    }
+    redir @httpget https://{host}{uri}
+  '';
+
+}
--- a/machines/thorite/restic.nix
+++ b/machines/thorite/restic.nix
@ -0,0 +1,51 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  sqliteBackup = fromPath: toPath: file: ''
+    mkdir -p ${toPath}
+    ${lib.getExe pkgs.sqlite} ${fromPath} ".backup '${toPath}/${file}'"
+  '';
+in
+{
+  sops.secrets = {
+    "restic/repo_url" = { };
+    "restic/repo_password" = { };
+  };
+
+  custom.restic = {
+    enable = true;
+    paths = [
+      "/backup/db"
+      "/backup/var/lib"
+    ];
+    backupPrepareCommand = [
+      ''
+        mkdir -p /backup/var
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r /var/lib /backup/var/lib
+      ''
+    ];
+    backupCleanupCommand = [
+      ''
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup/var/lib
+      ''
+    ];
+    btrfsRoots = [ ];
+  };
+
+  services.postgresqlBackup = lib.mkIf config.services.postgresql.enable {
+    enable = true;
+    compression = "zstd";
+    compressionLevel = 9;
+    location = "/backup/db/postgresql";
+  };
+
+  services.restic.backups.${config.networking.hostName} = {
+    extraBackupArgs = [
+      "--limit-upload=1024"
+    ];
+  };
+}
--- a/machines/thorite/secrets.yaml
+++ b/machines/thorite/secrets.yaml
@ -1,5 +1,8 @@
 grafana:
    oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str]
+restic:
+    repo_url: ENC[AES256_GCM,data:tc7wYRN20sHxATTZYEBpf6tNafzq9vcvqdUHYJDmJIArxprNd6WiyqPXowzbksZcEi5JwSwwJH/MYminnPGtrR8erWZg8OB3,iv:/z7mF58tMAviscFWHd4NJw7UZlq7Bzz+LU88J+kE9qg=,tag:i97FP4SmmNXOuxylkHhYCA==,type:str]
+    repo_password: ENC[AES256_GCM,data:o3MbXJRwR5UE9uCELN2ejQ==,iv:cYPNjJAV7H2BNCuFLDJoJvPk+CFvagXJwW9LRAGc0G0=,tag:qF6Di2W+8kESCRAphC/c0g==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -24,8 +27,8 @@ sops:
            M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx
            mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-28T17:02:03Z"
-    mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str]
+    lastmodified: "2024-12-03T08:18:54Z"
+    mac: ENC[AES256_GCM,data:jqSt34avoMfL9g3LmvjrPTzW4xGLgX70CXI8qk4isaLbZ8FkxjVU8QY1ot9GZnFEQWUkReSuGD4gFxi8TjetlNdx0zDPcv6zGJUSfcYpyKDCqGdyL/2x8xnYtI2pWINBZxR/2XxT3cus39FJdXVcz3l7KX4DvYvm8t/D9+r4ef0=,iv:KY/OTbDOOD/bBDTIuIk1ck7wDxLogo2EKeSOfOe4j5o=,tag:B17iF5O32KDZfctubpXCng==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/weilite/default.nix
+++ b/machines/weilite/default.nix
@ -52,9 +52,8 @@
          owner = "caddy";
          mode = "400";
        };
-        "immich/oauth_client_secret" = {
-          owner = "immich";
-          mode = "400";
+        "restic/localpass" = {
+          owner = "restic";
        };
      };
    };
@ -69,6 +68,10 @@
      };
    };

+    custom.monitoring = {
+      promtail.enable = true;
+    };
+
    systemd.mounts = [
      {
        what = "immich";
@ -105,43 +108,6 @@
      2222
    ];

-    services.immich = {
-      enable = true;
-      mediaLocation = "/mnt/XinPhotos/immich";
-      host = "127.0.0.1";
-      port = 3001;
-      openFirewall = true;
-      machine-learning.enable = true;
-      environment = {
-        IMMICH_MACHINE_LEARNING_ENABLED = "true";
-      };
-      database.enable = true;
-    };
-
-    custom.immich.jsonSettings = {
-      oauth = {
-        enabled = true;
-        issuerUrl = "https://auth.xinyang.life/oauth2/openid/immich/";
-        clientId = "immich";
-        clientSecret = {
-          _secret = config.sops.secrets."immich/oauth_client_secret".path;
-        };
-        scope = "openid email profile";
-        signingAlgorithm = "ES256";
-        storageLabelClaim = "email";
-        buttonText = "Login with Kanidm";
-        autoLaunch = true;
-        mobileOverrideEnabled = true;
-        mobileRedirectUri = "https://immich.xinyang.life:8000/api/oauth/mobile-redirect/";
-      };
-      passwordLogin = {
-        enabled = false;
-      };
-      newVersionCheck = {
-        enabled = false;
-      };
-    };
-
    services.dae = {
      enable = true;
      configFile = "/var/lib/dae/config.dae";
--- a/machines/weilite/secrets.yaml
+++ b/machines/weilite/secrets.yaml
@ -2,6 +2,8 @@ cloudflare_dns_token: ENC[AES256_GCM,data:m4euSkxxJmiMk9UPyeni/hwpl1W9A4MM0ssg71
 dnspod_dns_token: ENC[AES256_GCM,data:uZfr3g103amywxh3NMU+AkwuYb61svzyavvQ4rxJijIMIbfPvERrVNcyivoOrFWYXHpPWkhZFdU=,iv:mArVAcebW9i+u26GmQmfmJTsFkR4ZRMIisTqjpMYan8=,tag:Zsmv1Wzfi3+PHigjReToHQ==,type:str]
 immich:
    oauth_client_secret: ENC[AES256_GCM,data:EFs2hPjGMj0idwY3oQVIDTOIWkdwoAoAVjDQE9Z2eAKzUDH3grmYpYE+33V8d/Ux,iv:A9cjwFr/ZqltG62/N8MQ1LhdDbSIVVAqIPVB492zYJw=,tag:VTTtE697BZTVsI32UF53/w==,type:str]
+restic:
+    localpass: ENC[AES256_GCM,data:GIQAmkpDmGu4+sSG5/b5yQ==,iv:dcu6F8NnVjeQzEG2vM3fOV5owI0PWc86ts20UP3vN18=,tag:vsG8x062FG1pH5YNcAajeg==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -26,8 +28,8 @@ sops:
            V0thRjU4WGpQRGFpcnoxSjZTZHhTTkUKzNMHh9p7GUY3hL5XZ9S4x20CwaItsXFV
            RKujsFVVBd8Kuq/jyOCBTRCscuHI4LW/wYeZYHFEZFSTK2liAqspgw==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-13T12:02:54Z"
-    mac: ENC[AES256_GCM,data:c5p+B2mPCDyS/Q4QH4MkzCww6jFDhP8RfHqrKLf4e/8XuNEGfNmPKaeliZG26j1YQWRvFHiGQX3AMnQ3Q+fSRUQCVi5KV+KW7fADNIB3TiTT5hAFuynhiWWQSmIrWP0GGek3GDGi7OJ1PrFbxWP9bwaf+zBegiaUcWoTorJg7No=,iv:6MohNgPpq80eTUlf3RvPKsxdx69V0jl+/hrMxAPpPQE=,tag:BtWp1FChP2hdclbGl5W+vQ==,type:str]
+    lastmodified: "2024-12-03T05:59:51Z"
+    mac: ENC[AES256_GCM,data:0dLbfkm7fJvH5Mmct0/qHulg2AtDCeeeOgWMXfeGRUaX3GlLDiLga0zW4uNPDuahVecdh6ofvYfBOxFaGUdBCHk9vq5GzrwrzBNhqObWQ3AqVuq5rjqSxEKoFM4Eb5qoqaOefFzT/9qC94NDETTsHhjiEeIgd4fgSr2dazNiFPE=,iv:Ggw0FHzkrhKh5Uzo3seHGwwHsWW/tTAgAl0iIq9PVk4=,tag:rJvUI5/wsLJ01XyKmkRghw==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
-    version: 3.9.0
+    version: 3.9.1
--- a/machines/weilite/services/default.nix
+++ b/machines/weilite/services/default.nix
@ -3,5 +3,6 @@
    ./ocis.nix
    ./restic.nix
    ./media-download.nix
+    ./immich.nix
  ];
 }
--- a/machines/weilite/services/immich.nix
+++ b/machines/weilite/services/immich.nix
@ -0,0 +1,63 @@
+{
+  config,
+  ...
+}:
+let
+  user = config.systemd.services.immich-server.serviceConfig.User;
+  jsonSettings = {
+    oauth = {
+      enabled = true;
+      issuerUrl = "https://auth.xinyang.life/oauth2/openid/immich/";
+      clientId = "immich";
+      clientSecret = config.sops.placeholder."immich/oauth_client_secret";
+      scope = "openid email profile";
+      signingAlgorithm = "ES256";
+      storageLabelClaim = "email";
+      buttonText = "Login with Kanidm";
+      autoLaunch = true;
+      mobileOverrideEnabled = true;
+      mobileRedirectUri = "https://immich.xinyang.life:8000/api/oauth/mobile-redirect/";
+    };
+    passwordLogin = {
+      enabled = false;
+    };
+    image = {
+      extractEmbedded = true;
+    };
+    newVersionCheck = {
+      enabled = false;
+    };
+  };
+in
+{
+  config = {
+    sops.secrets."immich/oauth_client_secret" = { };
+
+    sops.templates."immich/config.json" = {
+      owner = user; # Read when running
+      content = builtins.toJSON jsonSettings;
+    };
+
+    systemd.services.immich-server = {
+      serviceConfig = {
+        Environment = "IMMICH_CONFIG_FILE=${config.sops.templates."immich/config.json".path}";
+      };
+    };
+
+    services.immich = {
+      enable = true;
+      mediaLocation = "/mnt/XinPhotos/immich";
+      host = "127.0.0.1";
+      port = 3001;
+      openFirewall = true;
+      machine-learning.enable = true;
+      environment = {
+        IMMICH_MACHINE_LEARNING_ENABLED = "true";
+      };
+      database.enable = true;
+    };
+
+    # https://github.com/NixOS/nixpkgs/pull/324127/files#r1723763510
+    services.immich.redis.host = "/run/redis-immich/redis.sock";
+  };
+}
--- a/machines/weilite/services/media-download.nix
+++ b/machines/weilite/services/media-download.nix
@ -13,6 +13,13 @@
    openFirewall = false;
  };

+  nixpkgs.config.permittedInsecurePackages = [
+    "aspnetcore-runtime-6.0.36"
+    "aspnetcore-runtime-wrapped-6.0.36"
+    "dotnet-sdk-6.0.428"
+    "dotnet-sdk-wrapped-6.0.428"
+  ];
+
  services.sonarr = {
    enable = true;
  };
--- a/machines/weilite/services/restic.nix
+++ b/machines/weilite/services/restic.nix
@ -35,6 +35,8 @@ in
  services.restic.backups = builtins.listToAttrs [
    (mkPrune "xin" "calcite")
    (mkPrune "xin" "massicot")
+    (mkPrune "xin" "biotite")
+    (mkPrune "xin" "thorite")
  ];

  networking.firewall.allowedTCPPorts = [ 8443 ];
--- a/modules/nixos/common-settings/auth.nix
+++ b/modules/nixos/common-settings/auth.nix
@ -9,8 +9,6 @@ let
  inherit (lib)
    mkIf
    mkEnableOption
-    mkOption
-    types
    ;

  cfg = config.commonSettings.auth;
@ -21,25 +19,43 @@ in
  };

  config = mkIf cfg.enable {
-    custom.kanidm-client = {
-      enable = true;
-      uri = "https://auth.xinyang.life";
-      asSSHAuth = {
-        enable = true;
-        allowedGroups = [ "linux_users" ];
+    services.kanidm = {
+      enableClient = true;
+      clientSettings = {
+        uri = "https://auth.xinyang.life";
+      };
+      enablePam = true;
+      unixSettings = {
+        pam_allowed_login_groups = [ "linux_users" ];
+        default_shell = "/bin/sh";
      };
-      sudoers = [ "xin@auth.xinyang.life" ];
    };

    services.openssh = {
+      enable = true;
+      authorizedKeysCommand = "/etc/ssh/auth %u";
+      authorizedKeysCommandUser = "kanidm-ssh-runner";
      settings = {
        PasswordAuthentication = false;
        KbdInteractiveAuthentication = false;
-        PermitRootLogin = "no";
-        GSSAPIAuthentication = "no";
-        KerberosAuthentication = "no";
+        PermitRootLogin = lib.mkForce "no";
      };
    };
+
+    environment.etc."ssh/auth" = {
+      mode = "0555";
+      text = ''
+        #!${pkgs.stdenv.shell}
+        ${pkgs.kanidm}/bin/kanidm_ssh_authorizedkeys $1
+      '';
+    };
+    users.groups.wheel.members = [ "xin@auth.xinyang.life" ];
+    users.groups.kanidm-ssh-runner = { };
+    users.users.kanidm-ssh-runner = {
+      isSystemUser = true;
+      group = "kanidm-ssh-runner";
+    };
+
    services.fail2ban.enable = true;

    security.sudo = {
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -7,15 +7,6 @@
    ./common-settings/mainland.nix
    ./disk-partitions
    ./restic.nix
-    ./vaultwarden.nix
    ./monitor
-    ./hedgedoc.nix
-    ./sing-box.nix
-    ./kanidm-client.nix
-    ./ssh-tpm-agent.nix # FIXME: Waiting for upstream merge
-    ./forgejo-actions-runner.nix
-    ./oidc-agent.nix
-    ./miniflux.nix
-    ./immich.nix
  ];
 }
--- a/modules/nixos/hedgedoc.nix
+++ b/modules/nixos/hedgedoc.nix
@ -1,78 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-
-with lib;
-
-let
-  cfg = config.custom.hedgedoc;
-in
-{
-  options = {
-    custom.hedgedoc = {
-      enable = mkEnableOption "HedgeDoc Markdown Editor";
-      domain = mkOption {
-        type = types.str;
-        default = "docs.example.com";
-        description = "Domain name of the HedgeDoc server";
-      };
-      caddy = mkOption {
-        type = types.bool;
-        default = true;
-        description = "Enable Caddy as reverse proxy";
-      };
-      mediaPath = mkOption {
-        type = types.path;
-        default = /var/lib/hedgedoc/uploads;
-        description = "Directory for storing medias";
-      };
-      oidc = {
-        enable = mkEnableOption "OIDC support for HedgeDoc";
-        baseURL = mkOption { type = types.str; };
-        authorizationURL = mkOption { type = types.str; };
-        tokenURL = mkOption { type = types.str; };
-        userProfileURL = mkOption { type = types.str; };
-      };
-      environmentFile = mkOption { type = types.path; };
-    };
-  };
-  config = mkIf cfg.enable {
-    services.hedgedoc = {
-      enable = true;
-      environmentFile = cfg.environmentFile;
-      settings = {
-        domain = cfg.domain;
-        protocolUseSSL = cfg.caddy;
-        uploadsPath = cfg.mediaPath;
-        path = "/run/hedgedoc/hedgedoc.sock";
-        email = false;
-        allowEmailRegister = false;
-        oauth2 = mkIf cfg.oidc.enable {
-          baseURL = cfg.oidc.baseURL;
-          authorizationURL = cfg.oidc.authorizationURL;
-          tokenURL = cfg.oidc.tokenURL;
-          userProfileURL = cfg.oidc.userProfileURL;
-          userProfileEmailAttr = "email";
-          userProfileUsernameAttr = "name";
-          userProfileDisplayNameAttr = "preferred_name";
-          scope = "openid email profile";
-          clientID = "$HEDGEDOC_CLIENT_ID";
-          clientSecret = "$HEDGEDOC_CLIENT_SECRET";
-        };
-        allowAnonymous = false;
-        defaultPermission = "private";
-      };
-    };
-    services.caddy = mkIf cfg.caddy {
-      enable = true;
-      virtualHosts."https://${cfg.domain}".extraConfig = ''
-        reverse_proxy unix/${config.services.hedgedoc.settings.path}
-      '';
-    };
-    users.users.caddy.extraGroups = mkIf cfg.caddy [ "hedgedoc" ];
-
-  };
-}
--- a/modules/nixos/immich.nix
+++ b/modules/nixos/immich.nix
@ -1,60 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  utils,
-  ...
-}:
-let
-  cfg = config.custom.immich;
-  upstreamCfg = config.services.immich;
-  settingsFormat = pkgs.formats.json { };
-  user = config.systemd.services.immich-server.serviceConfig.User;
-  group = config.systemd.services.immich-server.serviceConfig.Group;
-in
-{
-  options = {
-    custom.immich.jsonSettings = lib.mkOption {
-      type = lib.types.submodule {
-        freeformType = settingsFormat.type;
-      };
-      default = { };
-    };
-  };
-  config = {
-    /*
-      LoadCredential happens before preStart. We need to ensure the
-      configuration file exist, otherwise LoadCredential will fail.
-    */
-    systemd.tmpfiles.settings = lib.mkIf upstreamCfg.enable {
-      "10-etc-immich" = {
-        "/etc/immich" = {
-          d = {
-            inherit user group;
-            mode = "0700";
-          };
-        };
-        "/etc/immich/config.json" = {
-          "f+" = {
-            inherit user group;
-            mode = "0600";
-          };
-        };
-      };
-    };
-
-    systemd.services.immich-server = {
-      preStart = ''
-        umask 0077
-        ${utils.genJqSecretsReplacementSnippet cfg.jsonSettings "/etc/immich/config.json"}
-      '';
-      serviceConfig = {
-        LoadCredential = "config:/etc/immich/config.json";
-        Environment = "IMMICH_CONFIG_FILE=%d/config";
-      };
-    };
-
-    # https://github.com/NixOS/nixpkgs/pull/324127/files#r1723763510
-    services.immich.redis.host = "/run/redis-immich/redis.sock";
-  };
-}
--- a/modules/nixos/inbounds.nix
+++ b/modules/nixos/inbounds.nix
@ -1,134 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.sing-box-server;
-
-  secretFileType = lib.types.submodule { _secret = lib.types.path; };
-  singTls = {
-    enabled = true;
-    server_name = config.deployment.targetHost;
-    key_path = config.security.acme.certs.${config.deployment.targetHost}.directory + "/key.pem";
-    certificate_path =
-      config.security.acme.certs.${config.deployment.targetHost}.directory + "/cert.pem";
-  };
-in
-{
-  options = {
-    enable = lib.mkEnableOption "sing-box proxy server";
-    users = lib.types.listOf lib.types.submodule {
-      name = lib.mkOption {
-        type = lib.types.str;
-        default = "proxy";
-      };
-      password = lib.mkOption { type = secretFileType; };
-      uuid = lib.mkOption { type = secretFileType; };
-    };
-    wgOut = {
-      privKeyFile = lib.mkOption { type = lib.types.path; };
-      pubkey = lib.mkOption {
-        type = lib.types.str;
-        default = "bmXOC+F1FxEMF9dyiK2H5/1SUtzH0JuVo51h2wPfgyo=";
-      };
-    };
-    inbounds = {
-      trojan = {
-        enable = lib.mkOption {
-          type = lib.types.bool;
-          default = true;
-        };
-      };
-      tuic = {
-        enable = lib.mkOption {
-          type = lib.types.bool;
-          default = true;
-        };
-        ports = lib.mkOption {
-          type = lib.types.listOf lib.types.int;
-          default = lib.range 6311 6313;
-        };
-        directPorts = lib.mkOption {
-          type = lib.types.listOf lib.types.int;
-          default = [ 6314 ];
-        };
-      };
-    };
-  };
-  config = lib.mkIf cfg.enable {
-    services.sing-box = {
-      enable = true;
-      settings = {
-        dns = {
-          servers = [
-            {
-              address = "1.1.1.1";
-              detour = "wg-out";
-            }
-          ];
-        };
-        inbounds =
-          [
-            # TODO: Trojan and tuic enable
-            {
-              tag = "trojan-in";
-              type = "trojan";
-              listen = "::";
-              listen_port = 8080;
-              users = map (u: removeAttrs u [ "uuid" ]) cfg.users;
-              tls = singTls;
-            }
-          ]
-          ++ lib.forEach (cfg.tuic.ports ++ cfg.tuic.directPorts) (port: {
-            tag = "tuic-in" + toString port;
-            type = "tuic";
-            listen = "::";
-            listen_port = port;
-            congestion_control = "bbr";
-            users = cfg.users;
-            tls = singTls;
-          });
-        outbounds = [
-          {
-            type = "wireguard";
-            tag = "wg-out";
-            private_key = cfg.wgOut.privKeyFile;
-            local_address = [
-              "172.16.0.2/32"
-              "2606:4700:110:82ed:a443:3c62:6cbc:b59b/128"
-            ];
-            peers = [
-              {
-                public_key = cfg.wgOut.pubkey;
-                allowed_ips = [
-                  "0.0.0.0/0"
-                  "::/0"
-                ];
-                server = "162.159.192.1";
-                server_port = 500;
-              }
-            ];
-          }
-          {
-            type = "direct";
-            tag = "direct-out";
-          }
-          {
-            type = "dns";
-            tag = "dns-out";
-          }
-        ];
-        route = {
-          rules =
-            [
-              {
-                outbound = "dns-out";
-                protocol = "dns";
-              }
-            ]
-            ++ lib.forEach cfg.tuic.directPorts (port: {
-              inbound = "tuic-in" + toString port;
-              outbound = "direct-out";
-            });
-        };
-      };
-    };
-  };
-}
--- a/modules/nixos/kanidm-client.nix
+++ b/modules/nixos/kanidm-client.nix
@ -59,8 +59,6 @@ in
        PasswordAuthentication = false;
        KbdInteractiveAuthentication = false;
        PermitRootLogin = lib.mkForce "no";
-        GSSAPIAuthentication = "no";
-        KerberosAuthentication = "no";
      };
    };

--- a/modules/nixos/miniflux.nix
+++ b/modules/nixos/miniflux.nix
@ -1,165 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-let
-  inherit (lib)
-    mkEnableOption
-    mkPackageOption
-    mkOption
-    types
-    literalExpression
-    mkIf
-    mkDefault
-    ;
-  cfg = config.custom.miniflux;
-
-  defaultAddress = "localhost:8080";
-
-  pgbin = "${config.services.postgresql.package}/bin";
-  preStart = pkgs.writeScript "miniflux-pre-start" ''
-    #!${pkgs.runtimeShell}
-    ${pgbin}/psql "miniflux" -c "CREATE EXTENSION IF NOT EXISTS hstore"
-  '';
-in
-{
-  options = {
-    custom.miniflux = {
-      enable = mkEnableOption "miniflux";
-
-      package = mkPackageOption pkgs "miniflux" { };
-
-      oauth2SecretFile = mkOption { type = types.path; };
-
-      environment = mkOption {
-        type =
-          with types;
-          attrsOf (oneOf [
-            int
-            str
-          ]);
-      };
-
-      createDatabaseLocally = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Whether a PostgreSQL database should be automatically created and
-          configured on the local host. If set to `false`, you need provision a
-          database yourself and make sure to create the hstore extension in it.
-        '';
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.miniflux.enable = false;
-    custom.miniflux.environment = {
-      LISTEN_ADDR = mkDefault defaultAddress;
-      RUN_MIGRATIONS = mkDefault 1;
-      DATABASE_URL = lib.mkIf cfg.createDatabaseLocally "user=miniflux host=/run/postgresql dbname=miniflux";
-      OAUTH2_CLIENT_SECRET_FILE = "%d/oauth2_secret";
-      WATCHDOG = mkDefault 1;
-    };
-
-    services.postgresql = lib.mkIf cfg.createDatabaseLocally {
-      enable = true;
-      ensureUsers = [
-        {
-          name = "miniflux";
-          ensureDBOwnership = true;
-        }
-      ];
-      ensureDatabases = [ "miniflux" ];
-    };
-
-    systemd.services.miniflux-dbsetup = lib.mkIf cfg.createDatabaseLocally {
-      description = "Miniflux database setup";
-      requires = [ "postgresql.service" ];
-      after = [
-        "network.target"
-        "postgresql.service"
-      ];
-      serviceConfig = {
-        Type = "oneshot";
-        User = config.services.postgresql.superUser;
-        ExecStart = preStart;
-      };
-    };
-
-    systemd.services.miniflux = {
-      description = "Miniflux service";
-      wantedBy = [ "multi-user.target" ];
-      requires = lib.optional cfg.createDatabaseLocally "miniflux-dbsetup.service";
-      after =
-        [ "network.target" ]
-        ++ lib.optionals cfg.createDatabaseLocally [
-          "postgresql.service"
-          "miniflux-dbsetup.service"
-        ];
-
-      serviceConfig = {
-        Type = "notify";
-        ExecStart = lib.getExe cfg.package;
-        User = "miniflux";
-        DynamicUser = true;
-        LoadCredential = [ "oauth2_secret:${cfg.oauth2SecretFile}" ];
-        RuntimeDirectory = "miniflux";
-        RuntimeDirectoryMode = "0750";
-        WatchdogSec = 60;
-        WatchdogSignal = "SIGKILL";
-        Restart = "always";
-        RestartSec = 5;
-
-        # Hardening
-        CapabilityBoundingSet = [ "" ];
-        DeviceAllow = [ "" ];
-        LockPersonality = true;
-        MemoryDenyWriteExecute = true;
-        PrivateDevices = true;
-        PrivateUsers = true;
-        ProcSubset = "pid";
-        ProtectClock = true;
-        ProtectControlGroups = true;
-        ProtectHome = true;
-        ProtectHostname = true;
-        ProtectKernelLogs = true;
-        ProtectKernelModules = true;
-        ProtectKernelTunables = true;
-        ProtectProc = "invisible";
-        RestrictAddressFamilies = [
-          "AF_INET"
-          "AF_INET6"
-          "AF_UNIX"
-        ];
-        RestrictNamespaces = true;
-        RestrictRealtime = true;
-        RestrictSUIDSGID = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = [
-          "@system-service"
-          "~@privileged"
-        ];
-        UMask = "0077";
-      };
-
-      environment = lib.mapAttrs (_: toString) cfg.environment;
-    };
-    environment.systemPackages = [ cfg.package ];
-
-    security.apparmor.policies."bin.miniflux".profile = ''
-      include <tunables/global>
-      ${cfg.package}/bin/miniflux {
-        include <abstractions/base>
-        include <abstractions/nameservice>
-        include <abstractions/ssl_certs>
-        include "${pkgs.apparmorRulesFromClosure { name = "miniflux"; } cfg.package}"
-        r ${cfg.package}/bin/miniflux,
-        r @{sys}/kernel/mm/transparent_hugepage/hpage_pmd_size,
-        rw /run/miniflux/**,
-      }
-    '';
-  };
-}
--- a/modules/nixos/monitor/default.nix
+++ b/modules/nixos/monitor/default.nix
@ -11,6 +11,7 @@ let
    mkMerge
    types
    ;
+  inherit (config.my-lib.settings) ntfyUrl;
  cfg = config.custom.prometheus;

  mkRulesOption = mkOption {
@ -81,19 +82,9 @@ in
      ];
    }
    (mkIf cfg.enable {
-
-      services.tailscale = {
-        enable = true;
-        permitCertUid = config.services.caddy.user;
-        openFirewall = true;
-      };
-
-      services.caddy = {
-        enable = true;
-        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
-          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
-        '';
-      };
+      services.caddy.virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+      '';
      services.prometheus = mkIf cfg.enable {
        enable = true;
        port = 9091;
@ -121,12 +112,11 @@ in
                name = "ntfy";
                webhook_configs = [
                  {
-                    url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
-                      Alert {{.status}}
-                      {{range .alerts}}-----{{range $k,$v := .labels}}
+                    url = "${ntfyUrl}/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
+                      {{range .alerts}}[{{ if eq .status "resolved" }}✅ RESOLVED{{ else }}{{ if eq .status "firing" }}🔥 FIRING{{end}}{{end}}]{{range $k,$v := .labels}}
                      {{$k}}={{$v}}{{end}}
-                      {{end}}
-                    ''}";
+
+                      {{end}}''}";
                    send_resolved = true;
                  }
                ];
--- a/modules/nixos/monitor/exporters.nix
+++ b/modules/nixos/monitor/exporters.nix
@ -5,7 +5,8 @@
  ...
 }:
 let
-  inherit (lib) mkIf;
+  inherit (lib) mkIf concatStringsSep;
+  inherit (config.my-lib.settings) prometheusCollectors;
  cfg = config.custom.prometheus.exporters;
 in
 {
@ -71,7 +72,7 @@ in

    services.restic.server.prometheus = true;

-    # miniflux 
+    # miniflux
    sops.templates."miniflux_metrics_env" = {
      content = ''
        METRICS_COLLECTOR=1
@ -95,8 +96,28 @@ in
        metrics
      }

-      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
+      admin unix//var/run/caddy/admin.sock {
+        origins 127.0.0.1 ${config.networking.hostName}.coho-tet.ts.net:2019
      }
    '';
+
+    systemd.services.caddy.serviceConfig = {
+      RuntimeDirectory = "caddy";
+      RuntimeDirectoryMode = "0700";
+    };
+
+    services.tailscale = {
+      permitCertUid = config.services.caddy.user;
+      openFirewall = true;
+    };
+
+    services.caddy = {
+      virtualHosts."https://${config.networking.hostName}.coho-tet.ts.net:2019".extraConfig = ''
+        handle /metrics {
+        	reverse_proxy unix//var/run/caddy/admin.sock
+        }
+        respond 403
+      '';
+    };
  };
 }
--- a/modules/nixos/monitor/grafana.nix
+++ b/modules/nixos/monitor/grafana.nix
@ -1,5 +1,6 @@
 { config, lib, ... }:
 let
+  inherit (config.my-lib.settings) grafanaUrl idpUrl;
  cfg = config.custom.monitoring.grafana;
 in
 {
@ -13,17 +14,17 @@ in
        server = {
          http_addr = "127.0.0.1";
          http_port = 3003;
-          root_url = "https://grafana.xinyang.life";
-          domain = "grafana.xinyang.life";
+          root_url = grafanaUrl;
+          domain = lib.removePrefix "https://" grafanaUrl;
        };
        "auth.generic_oauth" = {
          enabled = true;
          name = "Kanidm";
          client_id = "grafana";
          scopes = "openid,profile,email,groups";
-          auth_url = "https://auth.xinyang.life/ui/oauth2";
-          token_url = "https://auth.xinyang.life/oauth2/token";
-          api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
+          auth_url = "${idpUrl}/ui/oauth2";
+          token_url = "${idpUrl}/oauth2/token";
+          api_url = "${idpUrl}/oauth2/openid/grafana/userinfo";
          use_pkce = true;
          use_refresh_token = true;
          allow_sign_up = true;
--- a/modules/nixos/monitor/loki.nix
+++ b/modules/nixos/monitor/loki.nix
@ -1,68 +1,159 @@
 {
+  pkgs,
  config,
  lib,
  ...
 }:
 let
  inherit (lib)
+    mkOption
    mkEnableOption
    mkIf
    mkMerge
+    types
+    literalExpression
+    ;
+  inherit (config.my-lib.settings)
+    alertmanagerPort
    ;
  cfg = config.custom.monitoring;
-  port-loki = 3100;
+  lokiPort = 3100;
 in
 {
  options = {
    custom.monitoring = {
-      loki.enable = mkEnableOption "loki";
+      loki = {
+        enable = mkEnableOption "loki";
+        rules = mkOption {
+          type = types.attrsOf (
+            types.submodule {
+              options = {
+                expr = mkOption {
+                  type = types.str;
+                  description = ''
+                    Loki alert expression.
+                  '';
+                  example = ''count_over_time({job=~"secure"} |="sshd[" |~": Failed|: Invalid|: Connection closed by authenticating user" | __error__="" [15m]) > 15'';
+                  default = null;
+                };
+                description = mkOption {
+                  type = types.str;
+                  description = ''
+                    Loki alert message.
+                  '';
+                  example = "Prometheus encountered value {{ $value }} with {{ $labels }}";
+                  default = null;
+                };
+                labels = mkOption {
+                  type = types.nullOr (types.attrsOf types.str);
+                  description = ''
+                    Additional alert labels.
+                  '';
+                  example = literalExpression ''
+                    { severity = "page" };
+                  '';
+                  default = { };
+                };
+                time = mkOption {
+                  type = types.str;
+                  description = ''
+                    Time until the alert is fired.
+                  '';
+                  example = "5m";
+                  default = "2m";
+                };
+              };
+            }
+          );
+          description = ''
+            Defines the loki rules.
+          '';
+          default = { };
+        };
+      };
      promtail.enable = mkEnableOption "promtail";
    };
  };

  config = mkMerge [
-    (mkIf cfg.loki.enable {
-      services.loki = {
-        enable = true;
-        configuration = {
-          auth_enabled = false;
-          server.http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
-          server.http_listen_port = port-loki;
-
-          common = {
-            ring = {
-              instance_addr = "${config.networking.hostName}.coho-tet.ts.net";
-              kvstore.store = "inmemory";
-            };
-            replication_factor = 1;
-            path_prefix = "/var/lib/loki";
-          };
-
-          schema_config.configs = [
+    (
+      let
+        rulerConfig = {
+          groups = [
            {
-              from = "2024-12-01";
-              store = "boltdb-shipper";
-              object_store = "filesystem";
-              schema = "v13";
-              index = {
-                prefix = "index_";
-                period = "24h";
-              };
+              name = "alerting-rules";
+              rules = lib.mapAttrsToList (name: opts: {
+                alert = name;
+                inherit (opts) expr labels;
+                for = opts.time;
+                annotations.description = opts.description;
+              }) cfg.loki.rules;
            }
          ];
+        };
+        rulerFile = pkgs.writeText "ruler.yml" (builtins.toJSON rulerConfig);
+      in
+      mkIf cfg.loki.enable {
+        services.loki = {
+          enable = true;
+          configuration = {
+            auth_enabled = false;
+            server.http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+            server.http_listen_port = lokiPort;

-          storage_config = {
-            filesystem.directory = "/var/lib/loki/chunks";
-          };
+            common = {
+              ring = {
+                instance_addr = "${config.networking.hostName}.coho-tet.ts.net";
+                kvstore.store = "inmemory";
+              };
+              replication_factor = 1;
+              path_prefix = "/var/lib/loki";
+            };

-          limits_config = {
-            reject_old_samples = true;
-            reject_old_samples_max_age = "168h";
-            allow_structured_metadata = false;
+            schema_config.configs = [
+              {
+                from = "2024-12-01";
+                store = "boltdb-shipper";
+                object_store = "filesystem";
+                schema = "v13";
+                index = {
+                  prefix = "index_";
+                  period = "24h";
+                };
+              }
+            ];
+
+            storage_config = {
+              filesystem.directory = "/var/lib/loki/chunks";
+            };
+
+            limits_config = {
+              reject_old_samples = true;
+              reject_old_samples_max_age = "168h";
+              allow_structured_metadata = false;
+            };
+
+            ruler = {
+              storage = {
+                type = "local";
+                local.directory = "${config.services.loki.dataDir}/rules";
+              };
+              rule_path = "${config.services.loki.dataDir}/rules-temp";
+              enable_api = true;
+              alertmanager_url = "http://127.0.0.1:${toString alertmanagerPort}";
+            };
          };
        };
-      };
-    })
+        systemd.tmpfiles.rules = [
+          "d /var/lib/loki 0700 loki loki - -"
+          "d /var/lib/loki/rules-temp 0700 loki loki - -"
+          "d /var/lib/loki/rules 0700 loki loki - -"
+          "d /var/lib/loki/rules/fake 0700 loki loki - -"
+          "L /var/lib/loki/rules/fake/ruler.yml - - - - ${rulerFile}"
+        ];
+        systemd.services.loki.restartTriggers = [ rulerFile ];
+      }
+    )
    (mkIf cfg.promtail.enable {
      services.promtail = {
        enable = true;
@ -78,7 +169,7 @@ in

          clients = [
            {
-              url = "http://thorite.coho-tet.ts.net:${toString port-loki}/loki/api/v1/push";
+              url = "http://thorite.coho-tet.ts.net:${toString lokiPort}/loki/api/v1/push";
            }
          ];

@ -150,17 +241,42 @@ in
              ];
            }
            # {
-            #   job_name = "caddy-access";
-            #   file_sd_configs = {
-            #     files = [
-            #       "/var/log/caddy/*.log"
-            #     ];
-            #     refresh_interval = "5m";
-            #   };
+            #   job_name = "caddy";
+            #   static_configs = [
+            #     {
+            #       targets = [ "localhost" ];
+            #       labels = {
+            #         job = "caddy";
+            #         __path__ = "/var/log/caddy/*log";
+            #         agent = "caddy-promtail";
+            #       };
+            #     }
+            #   ];
+            #   pipeline_stages = [
+            #     {
+            #       json = {
+            #         expressions = {
+            #           duration = "duration";
+            #           status = "status";
+            #         };
+            #       };
+            #     }
+            #     {
+            #       labels = {
+            #         duration = null;
+            #         status = null;
+            #       };
+            #     }
+            #   ];
            # }
          ];
        };
      };
+
+      services.caddy.logFormat = ''
+        format json
+        level INFO
+      '';
    })
  ];
 }
--- a/modules/nixos/restic.nix
+++ b/modules/nixos/restic.nix
@ -39,7 +39,7 @@ let
        echo "Creating snapshot for ${rootDir}"
        subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -o "${rootDir}" | ${awk} '{print $NF}')
        mkdir -p "${backupDir}"
-        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}" "${backupDir}/rootfs"
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}" "${backupDir}/rootDirectory"
        for subvol in $subvolumes; do
            ${continueIfInExclude}
            [[ /"$subvol" == "${backupDir}"* ]] && continue
--- a/modules/nixos/sing-box.nix
+++ b/modules/nixos/sing-box.nix
@ -1,87 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  utils,
-  ...
-}:
-let
-  cfg = config.custom.sing-box;
-  settingsFormat = pkgs.formats.json { };
-in
-{
-  options = {
-    custom.sing-box = {
-      enable = lib.mkEnableOption "sing-box";
-
-      package = lib.mkPackageOption pkgs "sing-box" { };
-
-      stateDir = lib.mkOption {
-        type = lib.types.path;
-        default = "/var/lib/sing-box";
-      };
-
-      configFile = {
-        urlFile = lib.mkOption { type = lib.types.path; };
-        name = lib.mkOption {
-          type = lib.types.str;
-          default = "config.json";
-        };
-        hash = lib.mkOption {
-          type = lib.types.str;
-          example = "9a304bcb87d4c3f1e50f6281f25dd78635255ebde06cd4d2555729ecda43aed4";
-        };
-      };
-
-      overrideSettings = lib.mkOption {
-        type = lib.types.submodule {
-          freeformType = settingsFormat.type;
-          options = {
-            route = {
-              geoip.path = lib.mkOption {
-                type = lib.types.path;
-                default = "${pkgs.sing-geoip}/share/sing-box/geoip.db";
-                defaultText = lib.literalExpression "\${pkgs.sing-geoip}/share/sing-box/geoip.db";
-                description = lib.mdDoc ''
-                  The path to the sing-geoip database.
-                '';
-              };
-              geosite.path = lib.mkOption {
-                type = lib.types.path;
-                default = "${pkgs.sing-geosite}/share/sing-box/geosite.db";
-                defaultText = lib.literalExpression "\${pkgs.sing-geosite}/share/sing-box/geosite.db";
-                description = lib.mdDoc ''
-                  The path to the sing-geosite database.
-                '';
-              };
-            };
-          };
-        };
-        default = { };
-      };
-    };
-  };
-  config = lib.mkIf cfg.enable {
-    networking.firewall.trustedInterfaces = [ "tun0" ];
-
-    systemd.packages = [ cfg.package ];
-
-    systemd.services.sing-box =
-      let
-        configFile = cfg.stateDir + "/${cfg.configFile.name}";
-      in
-      {
-        preStart = ''
-          umask 0077
-          mkdir -p /etc/sing-box
-          if ! [ -e ${configFile} ]; then
-            ${pkgs.curl}/bin/curl "$(${pkgs.coreutils}/bin/cat ${cfg.configFile.urlFile})" > '${configFile}'
-            test "${cfg.configFile.hash}" $(${pkgs.coreutils}/bin/sha256sum '${configFile}' | ${pkgs.coreutils}/bin/cut -d ' ' -f 1)
-          fi 
-          ${utils.genJqSecretsReplacementSnippet cfg.overrideSettings "/etc/sing-box/config.json"}
-          ${cfg.package}/bin/sing-box merge -c '${configFile}' -c /etc/sing-box/config.json /etc/sing-box/config.json
-        '';
-        wantedBy = [ "multi-user.target" ];
-      };
-  };
-}
--- a/note.md
+++ b/note.md
@ -1,3 +1,10 @@
 # nix-tree

 Demonstrate disk usage by nix-store path.
+
+## TODO
+- [x] change caddy admin to unix socket 
+- [ ] admin config persist = false
+- [x] synapse jmalloc
+- [ ] backup all directories under /var/lib/forgejo
+- [ ] collect caddy access logs with promtail (waiting for caddy v2.9.0 release after which log file mode can be set)
--- a/overlays/my-lib/default.nix
+++ b/overlays/my-lib/default.nix
@ -1,3 +1,11 @@
 {
+  mkSystemdDebug =
+    { lib, pkgs }:
+    {
+      ExecStart = lib.mkForce "${pkgs.tmux}/bin/tmux -S /tmp/tmux.socket new-session -s my-session -d";
+      ExecStop = lib.mkForce "${pkgs.tmux}/bin/tmux -S /tmp/tmux.socket kill-session -t my-session";
+      Type = "forking";
+    };
 }
 // (import ./prometheus.nix)
+// (import ./settings.nix)
--- a/overlays/my-lib/prometheus.nix
+++ b/overlays/my-lib/prometheus.nix
@ -28,17 +28,21 @@ in
    )
  );

-  mkCaddyScrapes = mkFunction (
+  mkCaddyScrapes = targets: [
    {
-      address,
-      port ? 2019,
-      ...
-    }:
-    {
-      job_name = "caddy_${address}";
-      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+      job_name = "caddy";
+      scheme = "https";
+      static_configs = map (
+        {
+          address,
+          port ? 2019,
+        }:
+        {
+          targets = [ "${address}${mkPort port}" ];
+        }
+      ) targets;
    }
-  );
+  ];

  mkCaddyRules = mkFunction (
    {
@ -63,17 +67,20 @@ in
    }
  );

-  mkNodeScrapes = mkFunction (
+  mkNodeScrapes = targets: [
    {
-      address,
-      port ? 9100,
-      ...
-    }:
-    {
-      job_name = "node_${address}";
-      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+      job_name = "node_exporter";
+      static_configs = map (
+        {
+          address,
+          port ? 9100,
+        }:
+        {
+          targets = [ "${address}${mkPort port}" ];
+        }
+      ) targets;
    }
-  );
+  ];

  mkNodeRules = mkFunction (
    {
@ -108,22 +115,10 @@ in
            description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
          };
        }
-        {
-          alert = "HighTransmitTraffic";
-          expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
-          for = "1m";
-          labels = {
-            severity = "warning";
-          };
-          annotations = {
-            summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
-            description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
-          };
-        }
        {
          alert = "NetworkTrafficExceedLimit";
-          expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
-          for = "0m";
+          expr = ''sum by(instance) (increase(node_network_transmit_bytes_total{device!="lo", device!~"tailscale.*", device!~"wg.*", device!~"br.*"}[30d])) > 322122547200'';
+          for = "1m";
          labels = {
            severity = "critical";
          };
@ -131,6 +126,66 @@ in
            summary = "Outbound network traffic exceed 300GB for last 30 day";
          };
        }
+        {
+          alert = "HighDiskUsage";
+          expr = ''(1 - node_filesystem_free_bytes{fstype!~"vfat|ramfs"} / node_filesystem_size_bytes) * 100 > 85'';
+          for = "5m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High disk usage on {{ $labels.instance }}";
+          };
+        }
+        {
+          alert = "DiskWillFull";
+          expr = ''predict_linear(node_filesystem_free_bytes{fstype!~"vfat|ramfs"}[1h], 12 * 3600) < (node_filesystem_size_bytes * 0.05)'';
+
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Disk usage will exceed 95% in 12 hours on {{ $labels.instance }}";
+            description = "Disk {{ $labels.mountpoint }} is predicted to exceed 92% usage within 12 hours at current growth rate";
+          };
+        }
+        {
+          alert = "HighSwapUsage";
+          expr = ''(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80'';
+          for = "5m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High swap usage on {{ $labels.instance }}";
+            description = "Swap usage is above 80% for 5 minutes\n Current value: {{ $value }}%";
+          };
+        }
+        {
+          alert = "OOMKillDetected";
+          expr = ''increase(node_vmstat_oom_kill[5m]) > 0'';
+          for = "1m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "OOM kill detected on {{ $labels.instance }}";
+            description = "Out of memory killer was triggered in the last 5 minutes";
+          };
+        }
+        {
+          alert = "HighMemoryUsage";
+          expr = ''(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90'';
+          for = "5m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High memory usage on {{ $labels.instance }}";
+            description = "Memory usage is above 90% for 5 minutes\n Current value: {{ $value }}%";
+          };
+        }
      ];
    }
  );
@ -152,6 +207,9 @@ in
      static_configs = [
        {
          targets = targetAddresses;
+          labels = {
+            from = hostAddress;
+          };
        }
      ];
      relabel_configs = [
@ -187,23 +245,25 @@ in
            severity = "warning";
          };
          annotations = {
-            summary = "High request latency on {{ $labels.instance }}";
-            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+            summary = "High request latency from {{ $labels.from }} to {{ $labels.instance }}";
+            description = "Request latency is above 0.5 seconds for the last 2 minutes.";
          };
        }
        {
          alert = "VeryHighProbeLatency";
-          expr = "probe_duration_seconds > 1";
+          expr = "probe_duration_seconds > 2";
          for = "3m";
          labels = {
            severity = "critical";
          };
          annotations = {
-            summary = "High request latency on {{ $labels.instance }}";
-            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+            summary = "Very high request latency from {{ $labels.from }} to {{ $labels.instance }}";
+            description = "Request latency is above 2 seconds for the last 2 minutes.";
          };
        }
      ];
    }
  );
+
+  # mkResticScrapes = mkFunction () ;
 }
--- a/overlays/my-lib/settings.nix
+++ b/overlays/my-lib/settings.nix
@ -0,0 +1,20 @@
+{
+  settings = {
+    alertmanagerPort = 9093;
+    idpUrl = "https://auth.xinyang.life";
+    gotosocialUrl = "https://gts.xiny.li";
+    minifluxUrl = "https://rss.xiny.li";
+    hedgedocDomain = "docs.xiny.li";
+    forgejoDomain = "git.xiny.li";
+    forgejoGitDomain = "git.xiny.li";
+    vaultwardenUrl = "https://vaultwarden.xiny.li";
+    ntfyUrl = "https://ntfy.xiny.li";
+    grafanaUrl = "https://grafana.xiny.li";
+    synapseUrl = "https://xiny.li";
+    synapseDelegateUrl = "https://synapse.xiny.li";
+
+    prometheusCollectors = [
+      "thorite.coho-tet.ts.net"
+    ];
+  };
+}
--- a/overlays/pkgs/oidc-agent/default.nix
+++ b/overlays/pkgs/oidc-agent/default.nix
@ -1,56 +0,0 @@
-{
-  lib,
-  stdenv,
-  fetchFromGitHub,
-  curl,
-  webkitgtk,
-  libmicrohttpd,
-  libsecret,
-  qrencode,
-  libsodium,
-  pkg-config,
-  help2man,
-}:
-
-stdenv.mkDerivation rec {
-  pname = "oidc-agent";
-  version = "5.1.0";
-
-  src = fetchFromGitHub {
-    owner = "indigo-dc";
-    repo = "oidc-agent";
-    rev = "v${version}";
-    sha256 = "sha256-cOK/rZ/jnyALLuhDM3+qvwwe4Fjkv8diQBkw7NfVo0c=";
-  };
-
-  buildInputs = [
-    pkg-config
-    help2man
-  ];
-  nativeBuildInputs = [
-    curl
-    webkitgtk
-    libmicrohttpd
-    libsecret
-    qrencode
-    libsodium
-  ];
-  enableParallelBuilding = true;
-
-  installPhase = ''
-    make -j $NIX_BUILD_CORES PREFIX=$out BIN_PATH=$out LIB_PATH=$out/lib \
-         install_bin install_lib install_conf
-  '';
-  postFixup = ''
-    # Override with patched binary to be used by help2man
-    cp -r $out/bin/* bin
-    make install_man PREFIX=$out
-  '';
-
-  meta = with lib; {
-    description = "oidc-agent for managing OpenID Connect tokens on the command line";
-    homepage = "https://github.com/indigo-dc/oidc-agent";
-    maintainers = [ ];
-    license = licenses.mit;
-  };
-}
Author	SHA1	Message	Date
xinyangli	9a21ab6621	modules/monitoring: clean up tailscale and caddy config	2024-12-05 17:36:01 +08:00
xinyangli	756357552a	biotite: move all services to biotite except kanidm	2024-12-05 13:28:22 +08:00
xinyangli	7bc5db676d	modules: clean up	2024-12-04 19:54:32 +08:00
xinyangli	241f7265d5	perf: my-lib under config	2024-12-04 18:19:22 +08:00
xinyangli	b667f8bf3f	thorite: scrape forgejo and hedgedoc	2024-12-04 16:34:51 +08:00
xinyangli	f260f7d847	calcite: drop tpm-ssh-agent and oidc-agent for now	2024-12-04 16:34:10 +08:00
xinyangli	ad9c205fc5	biotite: move services to new machines and new domain - related services: forgejo, miniflux, vaultwarden - moved from xinyang.life to xiny.li - clean up modules	2024-12-04 16:02:03 +08:00
xinyangli	947e97ce4e	thorite: fix loki alerts	2024-12-03 19:35:48 +08:00
xinyangli	d9a7b3d48c	my-lib/settings: manage settings shared globally	2024-12-03 16:49:28 +08:00
xinyangli	4169513ead	weilite: add insecure dotnet required by sonarr	2024-12-03 16:49:25 +08:00
xinyangli	bd4d7b5217	machines/thorite: add more scrapes, alerts; add restic backup;	2024-12-03 16:49:23 +08:00
xinyangli	bf74a01049	machines/biotite: add matrix-synapse and backup	2024-12-03 16:49:19 +08:00
xinyangli	83f7700949	modules/monitoring: add alert rules to loki	2024-12-03 16:49:17 +08:00
xinyangli	5b6f6ce735	home/firefox: fix sidebar and titlebar in userChrome.css	2024-12-03 16:49:14 +08:00
xinyangli	bd9f66238e	flake.lock: Update to 24.11 • Updated input 'catppuccin': 'github:catppuccin/nix/32359bf226fe874d3b7a0a5753d291a4da9616fe?narHash=sha256-0aIwr/RC/oe7rYkfJb47xjdEQDSNcqpFGsEa%2BEPlDEs%3D' (2024-11-10) → 'github:catppuccin/nix/a817009ebfd2cca7f70a77884e5098d0a8c83f8e?narHash=sha256-uX/9m0TbdhEzuWA0muM5mI/AaWcLiDLjCCyu5Qr9MRk%3D' (2024-11-30) • Updated input 'disko': 'github:nix-community/disko/869ba3a87486289a4197b52a6c9e7222edf00b3e?narHash=sha256-%2B4U2I2653JvPFxcux837ulwYS864QvEueIljUkwytsk%3D' (2024-11-26) → 'github:nix-community/disko/2814a5224a47ca19e858e027f7e8bff74a8ea9f1?narHash=sha256-2uMaVAZn7fiyTUGhKgleuLYe5%2BEAAYB/diKxrM7g3as%3D' (2024-11-30) • Updated input 'home-manager': 'github:nix-community/home-manager/1bd5616e33c0c54d7a5b37db94160635a9b27aeb?narHash=sha256-130gQ5k8kZlxjBEeLpE%2BSvWFgSOFgQFeZlqIik7KgtQ%3D' (2024-11-16) → 'github:nix-community/home-manager/c1fee8d4a60b89cae12b288ba9dbc608ff298163?narHash=sha256-dVmNuUajnU18oHzBQWZm1BQtANCHaqNuxTHZQ%2BGN0r8%3D' (2024-12-01) • Updated input 'my-nixvim': 'git+https://git.xinyang.life/xin/nixvim?ref=refs/heads/master&rev=a09d2b94efb5e2d801275a244eedaab0816f3702' (2024-11-03) → 'git+https://git.xinyang.life/xin/nixvim?ref=refs/heads/master&rev=a3709a89797ea094f82d38edeb4a538c07c8c3fa' (2024-11-30) • Updated input 'my-nixvim/nixvim': 'github:nix-community/nixvim/6f210158b03b01a1fd44bf3968165e6da80635ce?narHash=sha256-NByr7l7JetL9kIrdCOcRqBu%2BlAkruYXETp1DMiDHNQs%3D' (2024-11-02) → 'github:nix-community/nixvim/f11a877bcc1d66cc8bd7990c704f91c1e99c7d08?narHash=sha256-12OpSgbLDiKmxvBXwVracIfGI9FpjFyHpa1r0Ho%2BNFA%3D' (2024-11-13) • Updated input 'my-nixvim/nixvim/git-hooks': 'github:cachix/git-hooks.nix/af8a16fe5c264f5e9e18bcee2859b40a656876cf?narHash=sha256-W1MIJpADXQCgosJZT8qBYLRuZls2KSiKdpnTVdKBuvU%3D' (2024-10-30) → 'github:cachix/git-hooks.nix/d70155fdc00df4628446352fc58adc640cd705c2?narHash=sha256-fWPHyhYE6xvMI1eGY3pwBTq85wcy1YXqdzTZF%2B06nOg%3D' (2024-11-05) • Updated input 'my-nixvim/nixvim/home-manager': 'github:nix-community/home-manager/1743615b61c7285976f85b303a36cdf88a556503?narHash=sha256-AvCVDswOUM9D368HxYD25RsSKp%2B5o0L0/JHADjLoD38%3D' (2024-11-01) → 'github:nix-community/home-manager/60bb110917844d354f3c18e05450606a435d2d10?narHash=sha256-NjavpgE9/bMe/ABvZpyHIUeYF1mqR5lhaep3wB79ucs%3D' (2024-11-10) • Updated input 'my-nixvim/nixvim/nix-darwin': 'github:lnl7/nix-darwin/683d0c4cd1102dcccfa3f835565378c7f3cbe05e?narHash=sha256-qE/cYKBhzxHMtKtLK3hlSR3uzO1pWPGLrBuQK7r0CHc%3D' (2024-11-01) → 'github:lnl7/nix-darwin/5c74ab862c8070cbf6400128a1b56abb213656da?narHash=sha256-3Ftf9oqOypcEyyrWJ0baVkRpvQqroK/SVBFLvU3nPuc%3D' (2024-11-09) • Updated input 'my-nixvim/nixvim/nixpkgs': 'github:NixOS/nixpkgs/807e9154dcb16384b1b765ebe9cd2bba2ac287fd?narHash=sha256-l253w0XMT8nWHGXuXqyiIC/bMvh1VRszGXgdpQlfhvU%3D' (2024-10-29) → 'github:NixOS/nixpkgs/76612b17c0ce71689921ca12d9ffdc9c23ce40b2?narHash=sha256-IigrKK3vYRpUu%2BHEjPL/phrfh7Ox881er1UEsZvw9Q4%3D' (2024-11-09) • Updated input 'my-nixvim/nixvim/nuschtosSearch': 'github:NuschtOS/search/9e22bd742480916ff5d0ab20ca2522eaa3fa061e?narHash=sha256-8lklUZRV7nwkPLF3roxzi4C2oyLydDXyAzAnDvjkOms%3D' (2024-11-02) → 'github:NuschtOS/search/ef493352f9e1f051e01a55c062731503a6b36b4e?narHash=sha256-43yLsOm/wxBbfYSNDWVJeVv5Ij%2B23X3BIjFUfsdx/6M%3D' (2024-11-08) • Updated input 'my-nixvim/nixvim/nuschtosSearch/ixx': 'github:NuschtOS/ixx/65c207c92befec93e22086da9456d3906a4e999c?narHash=sha256-YcyJLvTmN6uLEBGCvYoMLwsinblXMkoYkNLEO4WnKus%3D' (2024-10-21) → 'github:NuschtOS/ixx/9fd01aad037f345350eab2cd45e1946cc66da4eb?narHash=sha256-EiOq8jF4Z/zQe0QYVc3%2BqSKxRK//CFHMB84aYrYGwEs%3D' (2024-10-26) • Updated input 'nix-index-database': 'github:Mic92/nix-index-database/bdba246946fb079b87b4cada4df9b1cdf1c06132?narHash=sha256-l9ryrx1Twh08a%2BgxrMGM9O/aZKEimZfa6sZVyPCImgI%3D' (2024-11-17) → 'github:Mic92/nix-index-database/6e0b7f81367069589a480b91603a10bcf71f3103?narHash=sha256-vy9Q41hBE7Zg0yakF79neVgb3i3PQMSMR7uHPpPywFE%3D' (2024-12-01) • Updated input 'nix-vscode-extensions': 'github:nix-community/nix-vscode-extensions/5cf92678e6799ce45442dee4c9cb8094843c7cfa?narHash=sha256-WwJqguc/5Q7HEwHlgDzDT8mtd8ZxInxZM2neJKC1oh8%3D' (2024-11-17) → 'github:nix-community/nix-vscode-extensions/e3a9b717e8327886d4ab6115f6989f4d1ef44e51?narHash=sha256-UhlyYYO84s36aSj0/xZdclY6CgwJSWPYtTHTOBuHodM%3D' (2024-12-02) • Updated input 'nixos-hardware': 'github:NixOS/nixos-hardware/672ac2ac86f7dff2f6f3406405bddecf960e0db6?narHash=sha256-UhWmEZhwJZmVZ1jfHZFzCg%2BZLO9Tb/v3Y6LC0UNyeTo%3D' (2024-11-16) → 'github:NixOS/nixos-hardware/fe01780d356d70fd119a19277bff71d3e78dad00?narHash=sha256-aQorWITXZu7b095UwnpUvcGt9dNJie/GO9r4hZfe2sU%3D' (2024-12-01) • Updated input 'nixpkgs': 'github:xinyangli/nixpkgs/b2644ed7258502987ad4a70cf8959bf5a26ce26d?narHash=sha256-nfqKsQhFCakM%2BeIKGf/JWu/g56rOPoGny10EZN8q7R0%3D' (2024-11-17) → 'github:xinyangli/nixpkgs/6273ca0a0fd51ac708a71e380c0cda97a72bbb07?narHash=sha256-JOIhbU0EPRXwFv1wCXGTkUZ9KnIcLxChvCqeV9hh63U%3D' (2024-12-02) • Updated input 'nixpkgs-stable': 'github:nixos/nixpkgs/c21b77913ea840f8bcf9adf4c41cecc2abffd38d?narHash=sha256-XUO0JKP1hlww0d7mm3kpmIr4hhtR4zicg5Wwes9cPMg%3D' (2024-11-15) → 'github:nixos/nixpkgs/7e1ca67996afd8233d9033edd26e442836cc2ad6?narHash=sha256-8qwPSE2g1othR1u4uP86NXxm6i7E9nHPyJX3m3lx7Q4%3D' (2024-12-01) • Updated input 'nur': 'github:nix-community/NUR/59740d792bea5caa547c9bc7ce366802ecfafb7f?narHash=sha256-GGp/rEfxRdi1BD9TlHoXxp2g9IuKDp0Jk7wYh1LacP8%3D' (2024-11-17) → 'github:nix-community/NUR/1844924bf1e7e5a98198eca17b6c27cc9a363b05?narHash=sha256-C8f6ekiZ4kP84JWLDrMigvnSK6RXQoxLEDoteXMx1yc%3D' (2024-12-02) • Updated input 'sops-nix': 'github:Mic92/sops-nix/47fc1d8c72dbd69b32ecb2019b5b648da3dd20ce?narHash=sha256-TGnMXCeXS924w9W6CvRFtUCUFr8E/RK138lHxU3vcw8%3D' (2024-11-17) → 'github:Mic92/sops-nix/c6134b6fff6bda95a1ac872a2a9d5f32e3c37856?narHash=sha256-m6/qwJAJYcidGMEdLqjKzRIjapK4nUfMq7rDCTmZajc%3D' (2024-12-02)	2024-12-03 16:49:01 +08:00
xinyangli	68852681f7	minor fix	2024-12-02 16:30:49 +08:00