modules/prometheus: refactor

modules/restic: snapshot all btrfs subvolumes
biotite,thorite: added
2024-11-30 11:25:35 +08:00 · 2024-11-28 12:07:22 +08:00 · 2024-11-27 17:48:58 +08:00 · 2024-11-27 17:43:56 +08:00 · 2024-11-26 17:37:16 +08:00 · 2024-11-26 17:36:16 +08:00
53 changed files with 1517 additions and 793 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -2,12 +2,13 @@ keys:
  - &xin age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
  - &host-calcite age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
  - &host-raspite age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
-  - &host-sgp-00 age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-  - &host-tok-00 age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
  - &host-la-00 age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
  - &host-massicot age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
  - &host-weilite age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml
-  - &host-hk-00 age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
+  - &host-hk-00 age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
+  - &host-fra-00 age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+  - &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+  - &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
 creation_rules:
  - path_regex: machines/calcite/secrets.yaml
    key_groups:
@ -24,24 +25,18 @@ creation_rules:
    - age:
      - *xin
      - *host-massicot
+  - path_regex: machines/thorite/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-thorite
  - path_regex: machines/dolomite/secrets/secrets.yaml
    key_groups:
    - age:
      - *xin
-      - *host-sgp-00
-      - *host-tok-00
      - *host-la-00
      - *host-hk-00
-  - path_regex: machines/dolomite/secrets/sgp-00.yaml
-    key_groups:
-    - age:
-      - *xin
-      - *host-sgp-00
-  - path_regex: machines/dolomite/secrets/tok-00.yaml
-    key_groups:
-    - age:
-      - *xin
-      - *host-tok-00
+      - *host-fra-00
  - path_regex: machines/dolomite/secrets/la-00.yaml
    key_groups:
    - age:
@ -52,6 +47,12 @@ creation_rules:
    - age:
      - *xin
      - *host-hk-00
+
+  - path_regex: machines/dolomite/secrets/fra-00.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-fra-00
  - path-regex: machines/weilite/secrets.yaml
    key_groups:
    - age:
@ -63,8 +64,6 @@ creation_rules:
      - *xin
      - *host-calcite
      - *host-raspite
-      - *host-sgp-00
-      - *host-tok-00
      - *host-la-00
      - *host-hk-00
      - *host-massicot
--- a/flake.lock
+++ b/flake.lock
@ -61,6 +61,26 @@
        "type": "github"
      }
    },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732645828,
+        "narHash": "sha256-+4U2I2653JvPFxcux837ulwYS864QvEueIljUkwytsk=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "869ba3a87486289a4197b52a6c9e7222edf00b3e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
    "flake-compat": {
      "flake": false,
      "locked": {
@ -596,6 +616,7 @@
      "inputs": {
        "catppuccin": "catppuccin",
        "colmena": "colmena",
+        "disko": "disko",
        "flake-utils": "flake-utils_2",
        "home-manager": "home-manager",
        "my-nixvim": "my-nixvim",
--- a/flake.nix
+++ b/flake.nix
@ -50,13 +50,18 @@
    catppuccin = {
      url = "github:catppuccin/nix";
    };
+
+    disko = {
+      url = "github:nix-community/disko";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
  };

  outputs =
    {
      self,
-      home-manager,
      nixpkgs,
+      home-manager,
      nixos-hardware,
      sops-nix,
      flake-utils,
@ -66,6 +71,7 @@
      nix-vscode-extensions,
      colmena,
      nix-index-database,
+      disko,
      ...
    }:
    let
@ -78,6 +84,7 @@
      overlayModule =
        { ... }:
        {
+          _module.args.my-lib = import ./overlays/my-lib;
          nixpkgs.overlays = [
            editorOverlay
            (import ./overlays/add-pkgs.nix)
@ -112,10 +119,21 @@
          ./machines/dolomite/bandwagon.nix
          ./machines/dolomite/common.nix
        ];
-        tok-00 = [
-          ./machines/dolomite/lightsail.nix
+        fra-00 = [
+          ./machines/dolomite/fra.nix
          ./machines/dolomite/common.nix
        ];
+        osmium = [
+          ./machines/osmium
+        ];
+        thorite = [
+          disko.nixosModules.disko
+          ./machines/thorite
+        ];
+        biotite = [
+          disko.nixosModules.disko
+          ./machines/biotite
+        ];
      };
      sharedColmenaModules = [
        deploymentModule
@ -157,7 +175,6 @@
        };
    in
    {
-      nixpkgs = nixpkgs;
      nixosModules.default = {
        imports = [
          ./modules/nixos
@ -169,7 +186,9 @@
      colmenaHive = colmena.lib.makeHive {
        meta = {
          # FIXME:
-          nixpkgs = import nixpkgs { system = "x86_64-linux"; };
+          nixpkgs = import nixpkgs {
+            system = "x86_64-linux";
+          };
        };

        massicot =
@ -184,20 +203,6 @@
            ] ++ sharedColmenaModules;
          };

-        tok-00 =
-          { ... }:
-          {
-            imports = nodeNixosModules.tok-00 ++ sharedColmenaModules;
-            nixpkgs.system = "x86_64-linux";
-            networking.hostName = "tok-00";
-            system.stateVersion = "23.11";
-            deployment = {
-              targetHost = "video01.namely.icu";
-              buildOnTarget = false;
-              tags = [ "proxy" ];
-            };
-          };
-
        la-00 =
          { ... }:
          {
@ -226,6 +231,20 @@
            };
          };

+        fra-00 =
+          { ... }:
+          {
+            imports = nodeNixosModules.fra-00 ++ sharedColmenaModules;
+            nixpkgs.system = "x86_64-linux";
+            networking.hostName = "fra-00";
+            system.stateVersion = "24.05";
+            deployment = {
+              targetHost = "fra-00.video.namely.icu";
+              buildOnTarget = false;
+              tags = [ "proxy" ];
+            };
+          };
+
        raspite =
          { ... }:
          {
@ -252,12 +271,29 @@
            };
            nixpkgs.system = "x86_64-linux";
          };
+        thorite =
+          { ... }:
+          {
+            imports = nodeNixosModules.thorite ++ sharedColmenaModules;
+            deployment = {
+              buildOnTarget = false;
+            };
+          };
+        biotite =
+          { ... }:
+          {
+            imports = nodeNixosModules.biotite ++ sharedColmenaModules;
+          };
      };

      nixosConfigurations = {
        calcite = mkNixos {
          hostname = "calcite";
        };
+
+        osmium = mkNixos {
+          hostname = "osmium";
+        };
      } // self.colmenaHive.nodes;

    }
--- a/home/xin/calcite.nix
+++ b/home/xin/calcite.nix
@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, lib, ... }:
 let
  homeDirectory = "/home/xin";
 in
@ -34,6 +34,7 @@ in
  };

  home.packages = with pkgs; [
+    resources
    thunderbird
    remmina
    qq
@ -56,11 +57,6 @@ in

  xdg.enable = true;

-  i18n.inputMethod = {
-    enabled = "fcitx5";
-    fcitx5.addons = with pkgs; [ fcitx5-rime ];
-  };
-
  custom-hm = {
    alacritty = {
      enable = true;
--- a/machines/biotite/default.nix
+++ b/machines/biotite/default.nix
@ -0,0 +1,32 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ./hardware-configurations.nix ];
+
+  networking.hostName = "biotite";
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.MACAddress = "b6:20:0d:9a:6c:34";
+    networkConfig = {
+      DHCP = "ipv4";
+      IPv6SendRA = true;
+    };
+    address = [ "2a03:4000:4a:148::1/64" ];
+  };
+
+  commonSettings = {
+    auth.enable = true;
+    autoupgrade.enable = true;
+  };
+
+  users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
+
+  system.stateVersion = "24.11";
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}
--- a/machines/biotite/hardware-configurations.nix
+++ b/machines/biotite/hardware-configurations.nix
@ -0,0 +1,22 @@
+{ config, modulesPath, ... }:
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vda";
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = config.diskPartitions.grubMbr;
+            root = config.diskPartitions.btrfs;
+          };
+        };
+      };
+    };
+  };
+}
--- a/machines/calcite/configuration.nix
+++ b/machines/calcite/configuration.nix
@ -105,6 +105,15 @@ in
    LC_TIME = "en_US.utf8";
  };

+  i18n.inputMethod = {
+    enable = true;
+    type = "fcitx5";
+    fcitx5 = {
+      addons = [ pkgs.fcitx5-rime ];
+      waylandFrontend = true;
+    };
+  };
+
  # ====== GUI ======

  programs.niri.enable = true;
@ -113,7 +122,7 @@ in

  catppuccin = {
    enable = true;
-    accent = "rosewater";
+    accent = "peach";
    flavor = "mocha";
  };

@ -173,6 +182,17 @@ in
          };
        };
      };
+      "keydous" = {
+        ids = [
+          "25a7:fa14"
+          "3151:4002"
+        ];
+        settings = {
+          main = {
+            capslock = "overload(control, esc)";
+          };
+        };
+      };
    };
  };

@ -315,26 +335,27 @@ in
  '';

  sops.secrets = {
-    restic_repo_calcite_password = {
+    "restic/repo_url" = {
      owner = "xin";
      sopsFile = ./secrets.yaml;
    };
-    restic_repo_calcite = {
+    "restic/repo_password" = {
      owner = "xin";
      sopsFile = ./secrets.yaml;
    };
-    sing_box_url = {
-      owner = "root";
-      sopsFile = ./secrets.yaml;
-    };
    "gitea/envfile" = {
      owner = "root";
      sopsFile = ./secrets.yaml;
    };
  };
-  custom.restic.enable = true;
-  custom.restic.repositoryFile = config.sops.secrets.restic_repo_calcite.path;
-  custom.restic.passwordFile = config.sops.secrets.restic_repo_calcite_password.path;
+
+  custom.restic = {
+    enable = true;
+    paths = [
+      "/backup/rootfs/var/lib"
+      "/backup/home"
+    ];
+  };

  custom.forgejo-actions-runner = {
    enable = false;
@ -347,8 +368,7 @@ in
  };

  custom.prometheus = {
-    enable = true;
-    exporters.blackbox.enable = true;
+    exporters.node.enable = true;
  };

  services.ollama = {
--- a/machines/calcite/secrets.yaml
+++ b/machines/calcite/secrets.yaml
@ -1,6 +1,6 @@
-restic_repo_calcite_password: ENC[AES256_GCM,data:9ALTQULAMyLY4FIxuVztf9r3,iv:fObBBeqpHAVYl8YUopz9fZd3YWB+0sc8l+sR12rmxb4=,tag:l3xDc2/cpQr38X/cd7qMXA==,type:str]
-restic_repo_calcite: ENC[AES256_GCM,data:ELvSvoBfulbsoMvRMt2bVo9KiNQAuHomblZcAwJ+g0tHELkq65kaaGwMsNy1AttBfiD7RrQsKifX/YTUGmuz1mDg0WqkV/Mv,iv:HKz96YgVahxh+t3AEqe09mTE01uT+VrUYt04H6zyS9g=,tag:llFeeN7ryTZI9gLlYIRhCg==,type:str]
-sing_box_url: ENC[AES256_GCM,data:2z2bDKdn51o1eaqhgE0pTg4FWcO8wcLNlnBZ69Q3Jm5GCxkXxsxN7DgqQvRVeakOHvaenQotF+nc6tlhKPsyzdQeG0yl3YYhGb9o3DkmpUjC6lalMSoiw1rSMVyBg4KYCWxmhR9iRurun62+5INGZwwHVqAjgWJhy/9+pdIFtgKyd/t0JhSU,iv:gIGbvRd88vZu3cVW7e4emZmmNO8QcubLrxS1sCwi4Co=,tag:AzLLtcA9jAbeuo6eWU6ilw==,type:str]
+restic:
+    repo_url: ENC[AES256_GCM,data:x/g1nZQ59SavVG+u5apNmBQ0Y5uQ9N0EKVh6qovqeP/Z7tmkudJtlBFD35C0ZidcQLAqTaZk1FFh8Ikjo4OcQSdTsx9BGvT4,iv:RQMOSEacDHXjYceBaAW4sFGk38vkijHuADcTS3DMxa8=,tag:769rLA2eRKjDrAaL/jERbA==,type:str]
+    repo_password: ENC[AES256_GCM,data:jqsIP1R5/yX8F0oYaSXACx6C,iv:KckzqctKLnmay+d30/Y4IttiASxYnMw6IHQrtwP2YdQ=,tag:L/Ij51UU1om48I8fd4iuwA==,type:str]
 gitea:
    envfile: ENC[AES256_GCM,data:CK+JNELuzjKgWnImuV4Euif3f3nNOACOrvc4NiIXs+q/F7QWrtpb3TK8/FrLNQk=,iv:QSDrlKJCBld2gDx/y1sT8anh37GhqSS2QZd2JJi5Yis=,tag:x5T6h59LBXhEyVwSr2dnuQ==,type:str]
 sops:
@ -27,8 +27,8 @@ sops:
            WGlLdXVoZlp3bEFXZjlMdG1VOUZDNUkKQ2NNTE3OsNUr2pOI7qeNFSCVkUIVRS+g
            FG5FbJJcFihXqr+Qo0nZkq+xq07vIia7mKoqyoIfkKwweiVzDKyrkQ==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-09T06:41:02Z"
-    mac: ENC[AES256_GCM,data:Hf8QYvRWxfs/JDOIAVnX5M0kv9Ktncfzq+Zf7i32TTsa94ShrgbUYVxQbRviOFDbjLfzswGKikLQ2EHLlH1KOFs7+mKKz5PKVAWJZnkAPa2oFXs41BcXLIg8sf4dhFxjzzhakeUX9Q0z4evJ1vMX06/VnnpHVSMhsnenSfBhWIA=,iv:uXKf2oYSb+0IWp6Ch0XuoFUIaUBiAW7Z8R9Z7LSdLvY=,tag:0VAcFakwCrHGZW5I8jmydA==,type:str]
+    lastmodified: "2024-11-28T03:55:19Z"
+    mac: ENC[AES256_GCM,data:VH7RnRT33ltsxycuSsUsM+64onQeClwQ3fIHUVQUyRJ6t7aJkBiGMQ80QtmwGE5CJTbq7LV4cis5Pq/f9vTb0SsY4tCSIgXNAE2zW2rjjQKjdHr+rnnKSJExJA+k2tL06Q/FUu+3SP7pVSaYBGQKb53UAbHsdJYbx00Ko6MzZ7U=,iv:EiYhbr6o4n3kGEEWKXeWmDPSb5hOvUhRH7N2ZLPRHmQ=,tag:BdI140bhvBW0bwQPpRYiRw==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/dolomite/claw.nix
+++ b/machines/dolomite/claw.nix
@ -27,18 +27,11 @@
    };

    fileSystems."/" = {
-      device = "/dev/disk/by-uuid/fe563e38-9a57-447a-ba57-c3e53ddd84ee";
+      device = "/dev/vda1";
      fsType = "ext4";
    };

-    swapDevices = [ ];
-
-    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-    # (the default) this is the recommended approach. When using systemd-networkd it's
-    # still possible to use this option, but it's recommended to use it in conjunction
-    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-    # networking.useNetworkd = false;
-
+    networking.useNetworkd = true;
    systemd.network.enable = true;
    systemd.network.networks."10-wan" = {
      matchConfig.MACAddress = "00:16:3e:0a:ec:45";
@ -47,7 +40,6 @@
        UseDNS = true;
      };
    };
-    # networking.interfaces.eth0.useDHCP = lib.mkDefault true;

    nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  };
--- a/machines/dolomite/common.nix
+++ b/machines/dolomite/common.nix
@ -1,4 +1,4 @@
-{ config, ... }:
+{ config, lib, ... }:
 {
  config = {
    sops = {
@ -21,15 +21,25 @@
        };
      };
    };
+    swapDevices = [
+      {
+        device = "/swapfile";
+        size = 2 * 1024;
+      }
+    ];

-    custom.prometheus = {
+    custom.prometheus.exporters = {
      enable = true;
-      exporters.blackbox.enable = true;
+      node.enable = true;
    };

+    services.tailscale.enable = true;
+
    commonSettings = {
      auth.enable = true;
-      proxyServer.enable = true;
+      proxyServer = {
+        enable = true;
+      };
    };
  };

--- a/machines/dolomite/ec2-metadata-fetcher.sh
+++ b/machines/dolomite/ec2-metadata-fetcher.sh
@ -1,66 +0,0 @@
-metaDir=/etc/ec2-metadata
-mkdir -m 0755 -p "$metaDir"
-rm -f "$metaDir/*"
-
-get_imds_token() {
-  # retry-delay of 1 selected to give the system a second to get going,
-  # but not add a lot to the bootup time
-  curl \
-    --silent \
-    --show-error \
-    --retry 3 \
-    --retry-delay 1 \
-    --fail \
-    -X PUT \
-    --connect-timeout 1 \
-    -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \
-    http://169.254.169.254/latest/api/token
-}
-
-preflight_imds_token() {
-  # retry-delay of 1 selected to give the system a second to get going,
-  # but not add a lot to the bootup time
-  curl \
-    --silent \
-    --show-error \
-    --retry 3 \
-    --retry-delay 1 \
-    --fail \
-    --connect-timeout 1 \
-    -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \
-    -o /dev/null \
-    http://169.254.169.254/1.0/meta-data/instance-id
-}
-
-try=1
-while [ $try -le 3 ]; do
-  echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..."
-  IMDS_TOKEN=$(get_imds_token) && break
-  try=$((try + 1))
-  sleep 1
-done
-
-if [ "x$IMDS_TOKEN" == "x" ]; then
-  echo "failed to fetch an IMDS2v token."
-fi
-
-try=1
-while [ $try -le 10 ]; do
-  echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..."
-  preflight_imds_token && break
-  try=$((try + 1))
-  sleep 1
-done
-
-echo "getting EC2 instance metadata..."
-
-get_imds() {
-  # --fail to avoid populating missing files with 404 HTML response body
-  # || true to allow the script to continue even when encountering a 404
-  curl --silent --show-error --fail --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@" || true
-}
-
-get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
-(umask 077 && get_imds -o "$metaDir/user-data" http://169.254.169.254/1.0/user-data)
-get_imds -o "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
-get_imds -o "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
--- a/machines/dolomite/fra.nix
+++ b/machines/dolomite/fra.nix
@ -0,0 +1,68 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-co
+# and may be overwritten by future invocations.  Please make chang
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 2 * 1024;
+    }
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "uhci_hcd"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+    "ahci"
+    "ata_piix"
+    "virtio_pci"
+    "xen_blkfront"
+    "vmw_pvscsi"
+  ];
+  boot.loader.grub = {
+    enable = true;
+    device = "/dev/sda";
+  };
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/sda1";
+    fsType = "ext4";
+  };
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.MACAddress = "00:16:3c:d2:7b:64";
+    networkConfig = {
+      DHCP = "no";
+      Gateway = "185.217.108.1";
+      DNSSEC = true;
+      DNSOverTLS = true;
+      DNS = [
+        "8.8.8.8#dns.google"
+        "8.8.4.4#dns.google"
+      ];
+    };
+    address = [ "185.217.108.59/24" ];
+  };
+
+  custom.prometheus.enable = false;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}
--- a/machines/dolomite/lightsail.nix
+++ b/machines/dolomite/lightsail.nix
@ -1,107 +0,0 @@
-{
-  config,
-  pkgs,
-  modulesPath,
-  ...
-}:
-let
-  cfg = config.ec2;
-in
-{
-  imports = [
-    "${modulesPath}/profiles/headless.nix"
-    # Note: While we do use the headless profile, we also explicitly
-    # turn on the serial console on ttyS0 below. This is because
-    # AWS does support accessing the serial console:
-    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-access-to-serial-console.html
-    "${modulesPath}/virtualisation/ec2-data.nix"
-    "${modulesPath}/virtualisation/amazon-init.nix"
-  ];
-
-  config = {
-    boot.loader.grub.device = "/dev/nvme0n1";
-
-    # from nixpkgs amazon-image.nix
-    assertions = [ ];
-
-    boot.growPartition = true;
-
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/nixos";
-      fsType = "ext4";
-      autoResize = true;
-    };
-
-    fileSystems."/boot" = {
-      # The ZFS image uses a partition labeled ESP whether or not we're
-      # booting with EFI.
-      device = "/dev/disk/by-label/ESP";
-      fsType = "vfat";
-    };
-
-    boot.extraModulePackages = [ config.boot.kernelPackages.ena ];
-    boot.initrd.kernelModules = [ "xen-blkfront" ];
-    boot.initrd.availableKernelModules = [ "nvme" ];
-    boot.kernelParams = [
-      "console=ttyS0,115200n8"
-      "random.trust_cpu=on"
-    ];
-
-    # Prevent the nouveau kernel module from being loaded, as it
-    # interferes with the nvidia/nvidia-uvm modules needed for CUDA.
-    # Also blacklist xen_fbfront to prevent a 30 second delay during
-    # boot.
-    boot.blacklistedKernelModules = [
-      "nouveau"
-      "xen_fbfront"
-    ];
-
-    boot.loader.grub.efiSupport = cfg.efi;
-    boot.loader.grub.efiInstallAsRemovable = cfg.efi;
-    boot.loader.timeout = 1;
-    boot.loader.grub.extraConfig = ''
-      serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
-      terminal_output console serial
-      terminal_input console serial
-    '';
-
-    systemd.services.fetch-ec2-metadata = {
-      wantedBy = [ "multi-user.target" ];
-      wants = [ "network-online.target" ];
-      after = [ "network-online.target" ];
-      path = [ pkgs.curl ];
-      script = builtins.readFile ./ec2-metadata-fetcher.sh;
-      serviceConfig.Type = "oneshot";
-      serviceConfig.StandardOutput = "journal+console";
-    };
-
-    # Amazon-issued AMIs include the SSM Agent by default, so we do the same.
-    # https://docs.aws.amazon.com/systems-manager/latest/userguide/ami-preinstalled-agent.html
-    services.amazon-ssm-agent.enable = true;
-
-    # Allow root logins only using the SSH key that the user specified
-    # at instance creation time.
-    services.openssh.enable = true;
-    services.openssh.settings.PermitRootLogin = "prohibit-password";
-
-    # Enable the serial console on ttyS0
-    systemd.services."serial-getty@ttyS0".enable = true;
-
-    # Creates symlinks for block device names.
-    services.udev.packages = [ pkgs.amazon-ec2-utils ];
-
-    # Force getting the hostname from EC2.
-    # networking.hostName = mkDefault "";
-
-    # Always include cryptsetup so that Charon can use it.
-    environment.systemPackages = [ pkgs.cryptsetup ];
-
-    # EC2 has its own NTP server provided by the hypervisor
-    services.timesyncd.enable = true;
-    services.timesyncd.servers = [ "169.254.169.123" ];
-
-    # udisks has become too bloated to have in a headless system
-    # (e.g. it depends on GTK).
-    services.udisks2.enable = false;
-  };
-}
--- a/machines/dolomite/secrets/fra-00.yaml
+++ b/machines/dolomite/secrets/fra-00.yaml
@ -0,0 +1,31 @@
+wg_private_key: ENC[AES256_GCM,data:wKZfXvNLh578VpWRkEGRiyDqEgJ9nHMGbliDP/FhX3ZqrPFLwuSF4D4tQgw=,iv:EU6OkblWfWuC7CPW0U0peYY6171TnhljqnszQhVJTFw=,tag:CBrZRXDSKYoqbx5x7wQ1Ew==,type:str]
+wg_ipv6_local_addr: ENC[AES256_GCM,data:A6oUJngb1sOAAVTbgeceEgTd3Ejs5WM4GmXLvJBif5nbQSgU67EHZpDv,iv:Yf9063C784jPjJICee/YEj6fgl357G9yfkz0haHJGss=,tag:++LbjP8AI0HdS/9rtMYDDg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBud0JBa3A0VTk5SHhpK0tq
+            THZEWkY0Yk1CNjVVOGVOckRncEJUT2MxdW13ClQ1ZXV1bVRTNnUvVVBmbVhTZ3Fa
+            Wm1iTDRYOUJ2MW04dkNlemxzdGk5ZXcKLS0tIEZpNXZINUxGN3ZyL2JTSzEwWWRY
+            NStaK1kyM0ozWVEyemNiN2pQZGNqRXMKOBwTvk4Sfl2BsB7foVqjw2GqPOdQwB+g
+            GUR09dG0z4/1rT3gPtDn88pjs2EZYWOMKq+BPGbz0951HFPOgPVB5g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0dnRMY3NSbWtyUlpXWFRJ
+            VGRKLzdjMStldmtVbW9ZM05QaWJzSWV0MndzCkdpWFppTC9DVnJDc0lDRkZLZ2F1
+            WDJGWjNMZEZraWg3VUpDVDVtOE9YanMKLS0tIEUvWmRwcTBkUzZIMEVjNGhqeXU5
+            YmxtM0hoWTIwY3RKcFkrdzdrRFYwVGcKhBIi6YKPROrTo/QTClmv/xFa8/KAsqJD
+            bA5gHAYJCu3WLpZqo1FXqMMX/4Jj3gtWq0jLDzQ0Xoma842dhJo4bw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-26T03:13:11Z"
+    mac: ENC[AES256_GCM,data:0cMicsi2HGDY28ZCRaIP9ynR0amfOSGYJtgJryWkbf8CVaDAmA51W5yXRxKYrdwd7T22wAWeFdKIeItm51FXtlPwUZyyWlOtfdq3JE/vKRPk711wuS30VY8rObW49A10jqZzM6sJ7jKVf3b1RvjCVqd5xuPLLczhg3Ft5jmAOtY=,iv:Vv80TdEYIEKQ5HExJHImDlEVfPO4k7THdN6XH8dLJ6Q=,tag:vNoA9vFRRrTOJbq93W0Ldw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.1
--- a/machines/dolomite/secrets/hk-00.yaml
+++ b/machines/dolomite/secrets/hk-00.yaml
@ -9,20 +9,20 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkNmVpY09ZNzhacDdpdVUr
-            SGc2NGNrRWlMMzE2RVNSN0tHTGNoeVhlWUFRCnpqNy9qMExKUFA0akFnNG1HS0h2
-            NXlmWkJMemJkam5oSEFaSENkRTRnczQKLS0tIGNha0RWbGFUWGpROEdoKy9WbC9n
-            WTUrUjMydHRHODN3TDhyakpHNG1hZjQKR3I8TwUDvvht9ck8YIplCjafhUdvxw7M
-            VNSjUoacKg0Uu5m777UlBpDdDXBwulrVryFxrKA0Q395+YRJ2Sg0wQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3UUhoT3hSSmhEM3ZteDhJ
+            VWdweThOUHVLVlNBUW5yVXpMOTN3UTNTbkd3CmlZL21yYWJvaW1VRGl5a0JCSVA5
+            RUdndFJqSnRCUllXTmNERkU2UHJIV3cKLS0tIFYvZkhpaDZEcVNCMzhZNzV2K0J4
+            QklidnA5Qmd0dGQ3UEFLdFBmaVNLajQKgw2HN9ksquyh+FV1c8OuThFSJlzGGgXM
+            HhmTFOrGBwLF2N8XGpVp+HcFnIWzjjK62sAVsomO/ak3Schg8283vg==
            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
+        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKZC9GU085TmV6b1FsdGFw
-            OEFJeVM1WFJib1lFM1luQmlQSGt3Ym1PaVVjCkd4TmhIcVB2Nk4xaHdwSVVHOGJJ
-            TVErNHZ1ZURKMmk2SzJUajFTV0tJSE0KLS0tIG5jVnZHNm55dncvaDdsWXNidDB1
-            TURVTjR3RUJzMmxmNVIyTk5rM0YvMU0KP3R78NlGqbRHmSn2WqanPq8Y9m+olBLO
-            2CTJI9QQfPACzz9KoEt5hlpqVpsgQT9CGDpyYEwXrFyxFY4QIh5NPw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3NU5RREplWEdsUkJiTVEx
+            QXdNUXlkdGdFQU5PZ2lwYTFmdHFUei9Fcnc0CjB1bjhuM3dhUXd3aEpwdlFMeith
+            aXFYV1hVVjd1SUwvNmhyeGNBMUZtT3cKLS0tIDFkQk9NN09zUFBuWm83R1hmWDZk
+            QWVGWVB5Rk1DcVBuSzFYRmRsOU5jL0kK0z3uFNq6dl67YepenXjoIkdV6sZaA7jB
+            QHe2qz1SzrQQ/7Lqf8aZNT6W5IwkNHpht27jetl119DerOhx6N58vQ==
            -----END AGE ENCRYPTED FILE-----
    lastmodified: "2024-10-17T10:52:20Z"
    mac: ENC[AES256_GCM,data:lxqZaTqs5d/b/iIZ7BbD2jYJq3fTIbFlbdwKbCAAiXJv8abxN6SjOKuecKEvkJ0Y7qf2e0Cl8lbRwSy5FJb9Wsl9O4LzF0KBu0lssnBtDuZujFldgxJSWB8kQ3vMsPQ+NbmRME3zdKazmuhEwS0h/O6L6KmnfHjtfnDpAjYD+MY=,iv:Xue3R2qGxiw5/hjr9dLiLqeKDTpnwAnx8v9M3qjz5EM=,tag:T67z1oCMoW/ApF6tFJL3dA==,type:str]
--- a/machines/dolomite/secrets/secrets.yaml
+++ b/machines/dolomite/secrets/secrets.yaml
@ -10,47 +10,47 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6dElZTXFjbzhNbE1OYmdP
-            M0JLVWMyOUpSMnQ1Q2hDc2VXVUxpblhDVUNjCmxGZXRsUmdWWjZPZGFhaDFHNnpx
-            YVVSWFl1YThwWENSVTdiWkRENlBhdDQKLS0tIGl0OWsrNXljLy9wejd4Q3JmTUFE
-            WGFaN21vb1EwTDdSOEFVSWlQZWR1Z1kKIy+vG42G/7hTJX9BNYXjy4GNnUEnzUgB
-            aRoLxgTpkTKezZiKkISQwEuFD8qC7aeQIV1kmGDpNK2uucJfFswvbQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmdCtZK2FVRTh3YVd3dm9m
+            ZWR5VVIvS3VOSGh2cmg2ZUFrYmNIdVNLSTNVCjlhVlJER1BZMlRUd1RkYnpvTE9F
+            bExGa1NBWWR0enBmUFJYVVA4UlI1cUkKLS0tIC8wa3FGRnFldVdTdkpBb2xQc3BD
+            cTlhNHplRUoyS3pxNnF0TVlFTy9kdzQK4kDSzSV4ZnELvCsajGwvsc/vzua2hbI1
+            Vht7rmZ8Dl4Y3xEIXG7XVnWK2GOblpqZ/eza1T6kWEkXp2uCdQnM6Q==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvNGE0Sk5lbXVNSjVQUTFF
-            VFFrVzJKczJwTWJJOEdKTVFhai9RWmJNSkJjCkNKQzRQWmcxTndIcERkMTFubi9K
-            SXVhbDhEMmRFRCtXdEVqMFdRbjQ3RTgKLS0tIGNIOWYzL0NUeklBRU5paEoyZ211
-            NDY5RDdwelMwVjVscHdOaGV2aTMwQUUKZaCo5jFlWxTsELGyQiY4CmcjdUcnBzOU
-            JzcWDMcODTo/yER/0jdPpdfvUWiGi12voIuqRJkON0x7d3X2d2Sexg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2LzI1M2orSDVyYTRRRnB6
-            d25oaHZSMWFUQ2lZTWxtVzFRSkxjd01tNjFZCmJHUWVGd2hYWVlpdk80WUxwM080
-            N0V1UW1hUC9GNWlPRCtuYUsxSzdmWUEKLS0tIEhSazVWeEpIVnoweWdnOEU2Q1hT
-            Yjl6bFRZS2RSRGpPWFdDS2lObCt0MGsKcFXy/2mLLlxY/vP+kCaeaR+9aBRL7ys1
-            x+HBAPqvcqvYk3MGBD9TpIW317RthDhEkY57GmtHgqIUsSLWsBgNdw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1dVV0U3kwSmdnTU1HcGpr
+            U2FKZVV1c1R6a3ovRGxoOUlrcUNWUUFHN25ZClBBTUZGeTc0Tkx1OXdaK1p6aWpr
+            aSsvN0ZDR1V3VnVrb1FBYzdHSTNXOVkKLS0tIFlSUk5LT1hVUUd1aVg1eVNTUURX
+            OXRVVmNRWEhmVXZkWC9HNTUyUTNrMlUK370K3D1vU97vHV9aGjYrFOIJzmOQAnzH
+            QR6XsOkM0FRvSkhTsEZ3qC4Wd2MTIyRzHYPKvZmz9LufIr1N/JFj1Q==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnL2NXTDNqWkYzQlVvM0xO
-            ZDk2RTFISHh3TmpTN2cxT3RTVnFUaURpK3dRCmJEVWJnNXdoT0JYYjBvcm4rSkZ0
-            QW5WeWhqWnZqaGlLRHphZW5PMUNZTDQKLS0tIGZFc2ZlREgwKysrNEhROUJzbHBU
-            TzhHdlV1bjduT1hlTVFMTmRtQmN0MFUKhCYQh5uVOjEj2kKSfSUVa8k35mqkDoTk
-            3CchebRciIR+w52d6uEsQove0248+OniG6bJ5ykkExLo1RzDQD7pBQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQT0YyeXI4d2o4V0lWUE4x
+            ZXZWWDFiakdqNlU5RWt6QUdxYVRSZzQyZkZBCi9Tdm5wRXB2cTYxdnVYRXJaS0d0
+            Lzg3VWpqQ1NOb1NTYXE4RGVRZVZoM1UKLS0tIFdGM01VU3FEc0ZyeEN3bVM1WEZq
+            M3BFa1hoWkQyRkJqSlZiTnBwQWphemcKLTAza2y96h+IyWB2EN6e4WIFQqeL5E7p
+            CDmHr+hSt6u9cr8C/etljxGMbKf9GqFOeuCyPugrJGdu4/qlR5iE0g==
            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
+        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhS0tDdThIRnNaZVZKanZY
-            bm1uV25nUzZITW5QY2Z2SkZtMFAvY1RVOWdrCnZMZ3F6dHd1TmhCMnZvbFhZYjJK
-            ZXRVUWNtVXVpOWFYWmdFQ2RZajlTQk0KLS0tIFJSYkxkelFTWkRYMjAvQ2lpTGRQ
-            bmE0bWg1U1ZkZHR4TEVtR0crbVZxdmcKeVUli/Tt4Xy4XxbUbFj9a4y6c9ZE/NjE
-            nCKLNYYPsZ/nS6qN3Pdetps4ziajJHUVmxCqNMHD+OoWqT6W8V/O6w==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFa1RHN2s4ajYzZmwvUlN2
+            c05SdERTTEhPRnJWOUF6TExIMnBEZkVMb1I4CkxBeTRQWmZEOGNrcFlGV2wrMkhI
+            QnAwSzZPaWNWbmdnZmFjZVJyRVdzN2cKLS0tIHVMU3Z6a1MrV3BVV1hqbEdYODJu
+            cGgvNU05eGx4alRNT2d5MWp6Q3lWZDAKQ+D1niMzaso/lQwdmepvACF8/SDEt2mQ
+            7nTRVJIpjGPTxO4ezcQWUGej+BSEnOoZno3epoIXLNlwDnHOAawTWQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNcHNReHZibVlrNUtncnl1
+            SzczRGVFdUNvcFdqeWpZUk5FL0hwOS9LT3l3CnFLdXozcUxXYUpjUXJZWEtjMXo3
+            d28reWd0Z1Y0NWdBTG1MTkRGSEphY2sKLS0tIGw5U3NiOU1DNitUd0x5SkJ3SHFj
+            RVpWNDNUb2d1SEZpQlFBK2tFVjFzU0kKtI7e+kkiBm1L/WzkBApRI8IIo3gHdrE1
+            fzR+sbYEHWf95iEmb/oGlH++TrFW/zRXEyWPAi4ORTs7s/Ql1UC4Wg==
            -----END AGE ENCRYPTED FILE-----
    lastmodified: "2024-11-22T05:51:19Z"
    mac: ENC[AES256_GCM,data:LPUb7YbELPsgYX+LvfuGdiNG1B5ZrvyRVZL9UiMHoJMDHaWpDGCQkT1bk5jEOewwFh+StK560UsPK4uW0+SqADJO80rmM3xfqlp5Ul2HJ8mU19l5C6FLpv2REIzhCp333rNJJlyhn3H6GZgMaWnjjLSX9XMOVaq1iz6Qt0P76SI=,iv://EdDr5D51RFuucq8gkei2RC7H2bkRYxP/7lerp9dtk=,tag:JrRQ19sZ0OAZouMgOiU/tQ==,type:str]
--- a/machines/dolomite/secrets/sgp-00.yaml
+++ b/machines/dolomite/secrets/sgp-00.yaml
@ -1,31 +0,0 @@
-wg_private_key: ENC[AES256_GCM,data:UjxZ3iC5hxVcVJdEUJ3+myaQ/6MvghDw6eKa2flSuxMwFS31WB7r3evjlI0=,iv:BjgXCps6gx1ISghEO42x5aKb+c/n0P1V8FMVlPxAyLY=,tag:IkxCkpyVre+sFoBlRSFpMA==,type:str]
-wg_ipv6_local_addr: ENC[AES256_GCM,data:ejDYuZjZCKcsvyUUKdXtxgBqWloIwYHmpc/YwCYq7O2thsxvOou6iSHf,iv:HDrMlec4svxHpZXMyRDzpdSKeJbTmkZPd98SHv2ZLhQ=,tag:LjpapuaJ6sl4USZC8xEU5w==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtUkpVa0dCSE1rTjZpaWR1
-            cjJjc25iOEV4TnhQUWE4SjI4QWVZYXdVcHdBCkIrNlVrV2xJRURVSG9sUHozeE5s
-            NitsV1MvcENZTHhmU01CSTRVNENXUFEKLS0tIGgxakQ2cGIzdzg5QzRoT3ZSaXUx
-            TkN5MkNTNitWMzVKZWdhNGRIZ3VNNDgKQ6lwM6EowuGOrskUpwD8VGirravE+e3/
-            Hkv5jLvvfVjmg0kvKlNRotTHrRUGV04JsbW7T9FfbKyYpmEb6oCrsg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjSUlkQzhYSGwyNnYvNHpQ
-            UktKOUZiYk56S0piVy9ZMFdYVFdsN1FEVkhVCnZETEM5MW84TlNpbm1hSXJtR2Yy
-            OEdrSi9lcmJOR2F1cUZqc0NyQjl4RDgKLS0tIHVLcnRicmVNd2MwVjB4cGFXTlBu
-            VkJCcXdqTkUzejNzSjIvV2YrVUc5Sm8KutTATsWJ5+yB/CFoGwTNshyI5LzwH4x5
-            i5EIIkVPdxSIHrXUp0j6+RPWMJvEOFIE3dVwxz+MxqqHqtmEny1WKA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-12-19T12:31:51Z"
-    mac: ENC[AES256_GCM,data:AY0/qJ1ZXv4mQlHnG3uY2zQ0FhIYjHBWKyXXpv2/Q6yZkuSu6nIQk039nd+nk7lczXy2cylTHyjYv5vDF6BJARhu4jeYov6yMqYR8ye8rXjZKcOfrN5yv7LV6jyuzBRBkCWTQsaoR8ycKHlrMe+vkAGu50epdAQjAG+Qv6RkBiM=,iv:dMi2CququdEIg+g8NMUb8ioKwEkUqTP+nrivtsUYUUY=,tag:drHI6oJUUwN3JadCHbWWkg==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.8.1
--- a/machines/dolomite/secrets/tok-00.yaml
+++ b/machines/dolomite/secrets/tok-00.yaml
@ -1,31 +0,0 @@
-wg_private_key: ENC[AES256_GCM,data:jz/03kP/dj625Jweu0MEw9aGm3Z3M1f43cZqGy2eElCIDhD78n+zZAqOM8c=,iv:fZxuvZLx97YyDoafQXbqVYjqRYzZq90PJiri9vdjwro=,tag:0A9sGnSl3y3gpEuvsdRtGg==,type:str]
-wg_ipv6_local_addr: ENC[AES256_GCM,data:W/uR+9kAKdXViAbZ0vEhC2eNwlzqX0x+LpzLrLCmQuVgRbZAtJCqfeE=,iv:pMZumU7fMV5MYX59hO7SEMLlG4m8DdPXeAiNgLxNzZk=,tag:xdGBpOBdWlc8Q9BDMv04sA==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkYTc2a2J3ZXRXTlRxQTAx
-            UjZVTTVPa0FjbS9jekI5eXhLOTdUQTlBS2pJCnVPL2Q1d05QR2NpTDVZeDFpSCs3
-            Yjh3aXkvdTBIOThVMGMzcUZmUWhtTjgKLS0tIFZvcy9zRVBRcDN0ekp0MEV5cEph
-            ZURTL3hnSHgwQTlSNklCK25icEM0SGsKq2jM6jXLfK38BgV0calwKLuHIcGw0zed
-            lT19Mt9jFsqmIkpJh1U9Ddpz63WND+7ruMdTZt6RWStIxww4m7pevg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSXBqdXcxUDNkS29Gd3ZY
-            dTA3bmNUVThtTFJtdnFpSjZQT01TTXhpYUc4CkFhcm14eUw1YXIyWEViMSsyc3pr
-            VUJqWWdHMCtoRGQ1T3dMQlg3ZTZ5dGMKLS0tIGQvbGpFZTdrVUFURE9tdENCZGwr
-            aDBKbitCTmhxNXVNRGh6TVBvbkNhTUEKIuj7B4RdueX7BfExgzVoo6YJf59GsUHa
-            j5kIJ5UeTqWEBGBaXcPjhHMEQjYqwSBsVz2XJmsxLhi8WxejLio8FA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-12-19T12:30:24Z"
-    mac: ENC[AES256_GCM,data:f+7+O2ZVSZJhr0fJlfO/AtZC2N/7gsNu1f4cnUoXYFb1wobyU6tLkbwGqeyIulokgIDAU5lJ62TJXAjybe+kE+PGtpr61KS7dyiO0LjzcT/X898oBYvJ9jtkuxDzKM4ve570U7ZmS7Jbxt2NJEkcBvSUJRdJHH5l0sDrvmW8cwY=,iv:mno6jVUDUWxsO353hbCqGub+NYfk0XFsWzmWCBUt6Gg=,tag:KOw7HTy+pETha5pzx5Pf8Q==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.8.1
--- a/machines/massicot/default.nix
+++ b/machines/massicot/default.nix
@ -24,10 +24,6 @@
      hedgedoc_env = {
        owner = "hedgedoc";
      };
-      grafana_cloud_api = {
-        owner = "prometheus";
-        sopsFile = ../secrets.yaml;
-      };
      grafana_oauth_secret = {
        owner = "grafana";
      };
@ -62,6 +58,8 @@
    hostName = "massicot";
  };

+  services.tailscale.enable = true;
+
  commonSettings = {
    auth.enable = true;
    nix = {
--- a/machines/massicot/networking.nix
+++ b/machines/massicot/networking.nix
@ -3,8 +3,10 @@
  networking.useNetworkd = true;
  systemd.network.networks."10-wan" = {
    matchConfig.MACAddress = "96:00:02:68:7d:2d";
-    networkConfig.DHCP = "ipv4";
-    networkConfig.Gateway = "fe80::1";
+    networkConfig = {
+      DHCP = "ipv4";
+      Gateway = "fe80::1";
+    };
    address = [
      "2a01:4f8:c17:345f::3/64"
    ];
--- a/machines/massicot/secrets.yaml
+++ b/machines/massicot/secrets.yaml
@ -7,8 +7,8 @@ miniflux:
 forgejo:
    env: ENC[AES256_GCM,data:TMeguXfanISeyvsay9SBqm3SSGKpp5nCkqhHblf0QHNzHWGQKwpORmWfOtVfgOh9qdDqq8wYBpXznmbvixjV,iv:IR/rMoAIvZCw9FURmau4+g8c3pvI9BRs7v1NJ5ia4jI=,tag:kjwf6RN5HN8I2sUhDcr4UQ==,type:str]
 restic:
-    repo: ENC[AES256_GCM,data:/vybkTU7LMWSlco9W2pJouU9wm4okXClSHXQMCA6SGIHWp4Ppl6C+jS4sNJALc6ntKzcEHyWO/R3JPjQKjZNH4YtrnNQp/ZY9g==,iv:gAvp6blg5JuBKzLw6YSgM1Uc24Aesov3ttCRXZXBvJw=,tag:pvH1y6BFOl7jIn/qQejUbQ==,type:str]
-    password: ENC[AES256_GCM,data:5eIIBtGtBFwcAQ+ZwTYOtg==,iv:3GEM8Imu0i1aTwwSspvz2EzwJOXUC/b15hzkFFuZ+YY=,tag:wscba+nMtshldgUtcEKnOw==,type:str]
+    repo_url: ENC[AES256_GCM,data:GMHbrjgwajnYSiqtoYaKiFT/aDWDwlzEkvMLPzYf7C9PvLr7T4zeWyAA9//8huldyxO3+nk6O9lR9ORZKZfb8/MYB7nRB03sZQ==,iv:6uBhsksOGDjoc13U2xWLz7I+0fzGRhnw0nStACqlnug=,tag:uhH28NYq+ly1bmCV/cpxkQ==,type:str]
+    repo_password: ENC[AES256_GCM,data:jRHNgOk5ChWdqMKsd/V4Xg==,iv:wrgF5pau/RylG1nmJYmvrZ02o67qkkT5PrZAQlXb6Qo=,tag:X0WVpMqi8xeoATss/sSPMA==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -33,8 +33,8 @@ sops:
            dnFBa0lDWWZtS1BHdzBoVzNTaGNkSEEKi/W1n7RT8NpTp00SBMwxsUJAPDhumJ/i
            V2VnaSNwouD3SswTcoBzqQpBP9XrqzjIYGke90ZODFQbMY9WDQ+O0g==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-30T07:19:35Z"
-    mac: ENC[AES256_GCM,data:WSGvA1RkChrD07Sf4BFVMbdTXQYxAHeGGQ52e+pnPh0lZPOzMc9sLDrBPqDK2OfrHC+hK8RC7FxQTGs6G/oBB4nUzIZPn9WycTiU5elwWDfktizH0gr3EJDm7Gs+bTWQpwdoJZGZ8XErK+yegCaKL5cSOSTlBBbQOnZfnoNBg5c=,iv:xyJRFfxHC2xV0ro4CbdOPau1zORxA64OqpvKr4aFZvQ=,tag:c9NA90d5WTK2pfxwoyOX5A==,type:str]
+    lastmodified: "2024-11-28T03:57:35Z"
+    mac: ENC[AES256_GCM,data:xjZrlwfWLtZNYfH+KiE2ICt9Jo4nx/LKaEYi/ECN/Od+ZTjety0V6RJ/RfmI6q3K1WMj0sAGc56hCZ0iOn25L8wK6dc14hZVoSwwbIiQ7hTQE5LcK+NbXNmy3r/YC855DHG9kE08eYGHdNcBbckZg3HhkHQ9UYS/Ox/QFFuBa5Q=,iv:N3AW+sr9ET3c/ArXr176haRewYFsfgsNn+hkC0MDJwA=,tag:SCikn+F8btuSBswV+oCdXg==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
-    version: 3.9.0
+    version: 3.9.1
--- a/machines/massicot/services.nix
+++ b/machines/massicot/services.nix
@ -43,10 +43,14 @@ in
    environmentFile = config.sops.secrets.hedgedoc_env.path;
  };

-  custom.prometheus = {
+  custom.prometheus.exporters = {
    enable = true;
-    exporters.blackbox.enable = true;
-    exporters.miniflux.enable = true;
+    blackbox = {
+      enable = true;
+    };
+    node = {
+      enable = true;
+    };
  };

  security.acme = {
--- a/machines/massicot/services/restic.nix
+++ b/machines/massicot/services/restic.nix
@ -12,22 +12,26 @@ let
 in
 {
  sops.secrets = {
-    "restic/repo" = {
+    "restic/repo_url" = {
      sopsFile = ../secrets.yaml;
    };
-    "restic/password" = {
+    "restic/repo_password" = {
      sopsFile = ../secrets.yaml;
    };
  };

  custom.restic = {
    enable = true;
-    repositoryFile = config.sops.secrets."restic/repo".path;
-    passwordFile = config.sops.secrets."restic/password".path;
    paths = [
      "/backup"
      "/mnt/storage"
    ];
+    backupPrepareCommand = [
+      (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite")
+      (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3")
+      (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite")
+      (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db")
+    ];
  };

  services.postgresqlBackup = {
@ -38,12 +42,6 @@ in
  };

  services.restic.backups.${config.networking.hostName} = {
-    backupPrepareCommand = builtins.concatStringsSep "\n" [
-      (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite")
-      (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3")
-      (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite")
-      (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db")
-    ];
    extraBackupArgs = [
      "--limit-upload=1024"
    ];
--- a/machines/minimal/default.nix
+++ b/machines/minimal/default.nix
@ -0,0 +1,25 @@
+{
+  lib,
+  ...
+}:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+  ];
+
+  boot.initrd.availableKernelModules =
+    [
+    ];
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}
--- a/machines/osmium/default.nix
+++ b/machines/osmium/default.nix
@ -0,0 +1,111 @@
+{
+  pkgs,
+  lib,
+  modulesPath,
+  ...
+}:
+{
+  imports = [
+    (modulesPath + "/installer/sd-card/sd-image.nix")
+    ./sd-image-aarch64-orangepi-r1plus.nix
+  ];
+
+  config = {
+    system.stateVersion = "24.05";
+
+    nixpkgs.system = "aarch64-linux";
+
+    boot.tmp.useTmpfs = false;
+    boot.kernelModules = [
+      "br_netfilter"
+      "bridge"
+    ];
+    boot.kernel.sysctl = {
+      "net.ipv4.ip_forward" = 1;
+      "net.ipv4.ip_nonlocal_bind" = 1;
+      "net.ipv6.conf.all.forwarding" = 1;
+      "net.ipv6.ip_nonlocal_bind" = 1;
+      "net.bridge.bridge-nf-call-ip6tables" = 1;
+      "net.bridge.bridge-nf-call-iptables" = 1;
+      "net.bridge.bridge-nf-call-arptables" = 1;
+      "fs.inotify.max_user_watches" = 524288;
+      "dev.i915.perf_stream_paranoid" = 0;
+      "net.ipv4.conf.all.rp_filter" = 0;
+      "vm.max_map_count" = 2000000;
+      "net.ipv4.conf.all.route_localnet" = 1;
+      "net.ipv4.conf.all.send_redirects" = 0;
+      "kernel.msgmnb" = 65536;
+      "kernel.msgmax" = 65536;
+      "net.ipv4.tcp_timestamps" = 0;
+      "net.ipv4.tcp_synack_retries" = 1;
+      "net.ipv4.tcp_syn_retries" = 1;
+      "net.ipv4.tcp_tw_recycle" = 1;
+      "net.ipv4.tcp_tw_reuse" = 1;
+      "net.ipv4.tcp_fin_timeout" = 15;
+      "net.ipv4.tcp_keepalive_time" = 1800;
+      "net.ipv4.tcp_keepalive_probes" = 3;
+      "net.ipv4.tcp_keepalive_intvl" = 15;
+      "net.ipv4.ip_local_port_range" = "2048 65535";
+      "fs.file-max" = 102400;
+      "net.ipv4.tcp_max_tw_buckets" = 180000;
+    };
+
+    commonSettings = {
+      nix.enableMirrors = true;
+      auth.enable = true;
+    };
+
+    documentation.enable = false;
+
+    time.timeZone = "Asia/Shanghai";
+    i18n = {
+      defaultLocale = "en_US.UTF-8";
+    };
+
+    environment.systemPackages = with pkgs; [
+      lsof
+      wget
+      curl
+      neovim
+      jq
+      iptables
+      ebtables
+      tcpdump
+      busybox
+      ethtool
+      socat
+      htop
+      iftop
+      lm_sensors
+    ];
+
+    programs.command-not-found.enable = false;
+
+    networking = {
+      useDHCP = false;
+      hostName = "osmium";
+    };
+
+    systemd.network = {
+      enable = true;
+      networks."lan" = {
+        matchConfig.Name = "enu1";
+        networkConfig.DHCP = "no";
+        linkConfig.RequiredForOnline = "no";
+      };
+      networks."wan" = {
+        matchConfig.Name = "end0";
+        networkConfig.DHCP = "yes";
+        linkConfig.RequiredForOnline = "yes";
+      };
+    };
+
+    services.dae = {
+      enable = true;
+      configFile = "/var/lib/dae/config.dae";
+    };
+
+    services.tailscale.enable = true;
+
+  };
+}
--- a/machines/osmium/sd-image-aarch64-orangepi-r1plus.nix
+++ b/machines/osmium/sd-image-aarch64-orangepi-r1plus.nix
@ -0,0 +1,44 @@
+{
+  config,
+  modulesPath,
+  lib,
+  pkgs,
+  ...
+}:
+let
+in
+{
+  imports = [
+    (modulesPath + "/profiles/base.nix")
+  ];
+
+  boot.loader.grub.enable = false;
+  boot.loader.generic-extlinux-compatible.enable = true;
+  boot.kernelPackages = pkgs.linuxPackages_latest;
+
+  boot.kernelParams = [
+    "earlycon"
+    "console=ttyS2,1500000"
+    "consoleblank=0"
+  ];
+  boot.supportedFilesystems = lib.mkForce [
+    "ext4"
+    "vfat"
+    "ntfs"
+  ];
+
+  sdImage = {
+    compressImage = false;
+    imageBaseName = "nixos-sd-image-orange-pi-r1-plus-lts";
+    firmwarePartitionOffset = 16;
+    populateFirmwareCommands = ''
+      echo "Install U-Boot: ${pkgs.ubootOrangePiR1LtsPackage}"
+      dd if=${pkgs.ubootOrangePiR1LtsPackage}/idbloader.img of=$img seek=64 conv=notrunc
+      dd if=${pkgs.ubootOrangePiR1LtsPackage}/u-boot.itb of=$img seek=16384 conv=notrunc
+    '';
+    populateRootCommands = ''
+      mkdir -p ./files/boot
+      ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d ./files/boot
+    '';
+  };
+}
--- a/machines/thorite/default.nix
+++ b/machines/thorite/default.nix
@ -0,0 +1,41 @@
+{
+  imports = [
+    ./hardware-configurations.nix
+    ./monitoring.nix
+  ];
+
+  config = {
+    networking.hostName = "thorite";
+    networking.useNetworkd = true;
+    systemd.network.enable = true;
+    systemd.network.networks."10-wan" = {
+      matchConfig.MACAddress = "00:51:d3:21:f3:28";
+      networkConfig = {
+        DHCP = "no";
+        Gateway = "23.165.200.1";
+        DNSSEC = true;
+        DNSOverTLS = true;
+        DNS = [
+          "8.8.8.8#dns.google"
+          "8.8.4.4#dns.google"
+        ];
+      };
+      address = [ "23.165.200.99/24" ];
+    };
+
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+
+    commonSettings = {
+      auth.enable = true;
+      autoupgrade.enable = true;
+    };
+
+    nixpkgs.system = "x86_64-linux";
+    system.stateVersion = "24.11";
+
+    users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
+  };
+}
--- a/machines/thorite/hardware-configurations.nix
+++ b/machines/thorite/hardware-configurations.nix
@ -0,0 +1,40 @@
+{ config, modulesPath, ... }:
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/sda";
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = config.diskPartitions.grubMbr;
+            root = config.diskPartitions.btrfs;
+          };
+        };
+      };
+    };
+  };
+  disko.devices.disk.main.imageSize = "10G";
+
+  boot.initrd.availableKernelModules = [
+    "uhci_hcd"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+    "ahci"
+    "ata_piix"
+    "virtio_pci"
+    "xen_blkfront"
+    "vmw_pvscsi"
+  ];
+
+  boot.loader.grub = {
+    enable = true;
+  };
+
+  boot.kernelModules = [ "kvm-intel" ];
+}
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@ -0,0 +1,100 @@
+{ config, my-lib, ... }:
+with my-lib;
+{
+  config = {
+    sops = {
+      defaultSopsFile = ./secrets.yaml;
+      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+      secrets = {
+        "grafana/oauth_secret" = {
+          owner = "grafana";
+        };
+      };
+    };
+
+    custom.monitoring = {
+      grafana.enable = true;
+    };
+
+    services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =
+      with config.services.grafana.settings.server; ''
+        reverse_proxy http://${http_addr}:${toString http_port}
+      '';
+
+    custom.prometheus = {
+      enable = true;
+      exporters = {
+        enable = true;
+        blackbox.enable = true;
+        node.enable = true;
+      };
+      ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]);
+    };
+
+    services.prometheus.scrapeConfigs =
+      let
+        probeList = [
+          "la-00.video.namely.icu:8080"
+          "fre-00.video.namely.icu:8080"
+          "hk-00.video.namely.icu:8080"
+          "49.13.13.122:443"
+          "45.142.178.32:22"
+          "home.xinyang.life:8000"
+        ];
+      in
+      (mkScrapes [
+        {
+          name = "immich";
+          scheme = "http";
+          address = "weilite.coho-tet.ts.net";
+          port = 8082;
+        }
+        {
+          name = "gotosocial";
+          address = "xinyang.life";
+        }
+        {
+          name = "miniflux";
+          address = "rss.xinyang.life";
+        }
+        {
+          name = "ntfy";
+          address = "ntfy.xinyang.life";
+        }
+        {
+          name = "grafana-eu";
+          address = "grafana.xinyang.life";
+        }
+      ])
+      ++ (mkCaddyScrapes [
+        { address = "thorite.coho-tet.ts.net"; }
+      ])
+      ++ (mkNodeScrapes [
+        { address = "thorite.coho-tet.ts.net"; }
+        { address = "massicot.coho-tet.ts.net"; }
+        { address = "weilite.coho-tet.ts.net"; }
+        { address = "hk-00.coho-tet.ts.net"; }
+        { address = "la-00.coho-tet.ts.net"; }
+        { address = "fra-00.coho-tet.ts.net"; }
+      ])
+      ++ (mkBlackboxScrapes [
+        {
+          hostAddress = "thorite.coho-tet.ts.net";
+          targetAddresses = probeList;
+        }
+        {
+          hostAddress = "massicot.coho-tet.ts.net";
+          targetAddresses = probeList;
+        }
+        {
+          hostAddress = "weilite.coho-tet.ts.net";
+          targetAddresses = [
+            "la-00.video.namely.icu:8080"
+            "fre-00.video.namely.icu:8080"
+            "hk-00.video.namely.icu:8080"
+          ];
+        }
+      ]);
+
+  };
+}
--- a/machines/thorite/secrets.yaml
+++ b/machines/thorite/secrets.yaml
@ -0,0 +1,31 @@
+grafana:
+    oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoTXRtTlRES3V4MGhZaGdr
+            aXJ4UFNDT0Nrb0ZuWEkxUEFDU2orbzNBSVhVCkh2VitqMGwwOVdhMFJIeWU1eTgw
+            UVdxY0tLVDJNVnRnQmMyS0FPYS9LVmMKLS0tIEZaMTdIMU5SQUkxL2NFK2Jtbm9v
+            YVR3RHpDR3F2aFlCWGd5TjNOV2p4YzgK8OKpwcvTK/0j+kQCo0+8n6sQ5Pu9t9xZ
+            lPWeUGk1BudsyCqgIZWF5iXfu1pJnYq1XEAM0ttJl402xKeqIovM0Q==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIYk1NTlhsYW8xbFppNTBE
+            WlJmNzhnclBoVENXa1cvcHY5NGdRZVAzV0FJClpsTHpTeG9CK3J2ZFEreG1BTWpG
+            WjdaYzlLQnU0LzJLSDBZZ2pvOWdvSEkKLS0tIExRT0p1aCttZG5MMW14emJmRk5w
+            M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx
+            mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-28T17:02:03Z"
+    mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.1
--- a/machines/weilite/default.nix
+++ b/machines/weilite/default.nix
@ -16,7 +16,6 @@
    networking.hostName = "weilite";
    commonSettings = {
      auth.enable = true;
-      autoupgrade.enable = true;
      nix = {
        enable = true;
        enableMirrors = true;
@ -61,8 +60,14 @@
      };
    };

-    custom.prometheus = {
+    custom.prometheus.exporters = {
      enable = true;
+      blackbox = {
+        enable = true;
+      };
+      node = {
+        enable = true;
+      };
    };

    systemd.mounts = [
--- a/machines/weilite/services/media-download.nix
+++ b/machines/weilite/services/media-download.nix
@ -1,6 +1,23 @@
+{ pkgs, ... }:
 {
  services.jackett = {
    enable = true;
+    package = pkgs.jackett.overrideAttrs {
+      src = pkgs.fetchFromGitHub {
+        owner = "jackett";
+        repo = "jackett";
+        rev = "v0.22.998";
+        hash = "sha256-CZvgDWxxIAOTkodgmFNuT3VDW6Ln4Mz+Ki7m91f0BgE=";
+      };
+    };
    openFirewall = false;
  };
+
+  services.sonarr = {
+    enable = true;
+  };
+
+  services.radarr = {
+    enable = true;
+  };
 }
--- a/modules/home-manager/zellij.nix
+++ b/modules/home-manager/zellij.nix
@ -18,6 +18,17 @@ in
    };
    xdg.configFile."zellij/config.kdl".text = ''
      keybinds {
+          shared {
+            bind "F1" { GoToTab 1; SwitchToMode "Normal"; }
+            bind "F2" { GoToTab 2; SwitchToMode "Normal"; }
+            bind "F3" { GoToTab 3; SwitchToMode "Normal"; }
+            bind "F4" { GoToTab 4; SwitchToMode "Normal"; }
+            bind "F5" { GoToTab 5; SwitchToMode "Normal"; }
+            bind "F6" { GoToTab 6; SwitchToMode "Normal"; }
+            bind "F7" { GoToTab 7; SwitchToMode "Normal"; }
+            bind "F8" { GoToTab 8; SwitchToMode "Normal"; }
+            bind "F9" { GoToTab 9; SwitchToMode "Normal"; }
+          }
          shared_except "pane" "locked" {
            bind "Ctrl b" { SwitchToMode "Pane"; }
          }
--- a/modules/nixos/common-settings/proxy-server.nix
+++ b/modules/nixos/common-settings/proxy-server.nix
@ -1,7 +1,6 @@
 {
  config,
  lib,
-  pkgs,
  ...
 }:

@ -10,6 +9,7 @@ let
    mkIf
    mkEnableOption
    mkOption
+    mkDefault
    types
    ;

@ -32,7 +32,9 @@ let
            tag = "sg0";
            type = "trojan";
            listen = "::";
-            listen_port = 8080;
+            listen_port = cfg.trojan.port;
+            tcp_multi_path = true;
+            tcp_fast_open = true;
            users = [
              {
                name = "proxy";
@ -63,51 +65,77 @@ let
          ];
          tls = singTls;
        });
-      outbounds = [
-        {
-          type = "wireguard";
-          tag = "wg-out";
-          private_key = {
-            _secret = config.sops.secrets.wg_private_key.path;
-          };
-          local_address = [
-            "172.16.0.2/32"
-            { _secret = config.sops.secrets.wg_ipv6_local_addr.path; }
-          ];
-          peers = [
-            {
-              public_key = "bmXOC+F1FxEMF9dyiK2H5/1SUtzH0JuVo51h2wPfgyo=";
-              allowed_ips = [
-                "0.0.0.0/0"
-                "::/0"
-              ];
-              server = "162.159.192.1";
-              server_port = 500;
-            }
-          ];
-        }
-        {
-          type = "direct";
-          tag = "direct";
-        }
-      ];
-      route = {
-        rules = [
+      outbounds =
+        # warp outbound goes first to make it default outbound
+        (lib.optionals (cfg.warp.onTuic or cfg.warp.onTrojan) [
          {
-            inbound = "sg0";
-            outbound = "direct";
+            type = "wireguard";
+            tag = "wg-out";
+            private_key = {
+              _secret = config.sops.secrets.wg_private_key.path;
+            };
+            local_address = [
+              "172.16.0.2/32"
+              { _secret = config.sops.secrets.wg_ipv6_local_addr.path; }
+            ];
+            peers = [
+              {
+                public_key = "bmXOC+F1FxEMF9dyiK2H5/1SUtzH0JuVo51h2wPfgyo=";
+                allowed_ips = [
+                  "0.0.0.0/0"
+                  "::/0"
+                ];
+                server = "162.159.192.1";
+                server_port = 500;
+              }
+            ];
          }
+        ])
+        ++ [
+
          {
-            inbound = "sg4";
-            outbound = "direct";
+            type = "direct";
+            tag = "direct";
          }
        ];
+      route = {
+        rules =
+          [
+            {
+              inbound = "sg4";
+              outbound = "direct";
+            }
+          ]
+          ++ (lib.optionals (!cfg.warp.onTuic) (
+            lib.forEach (lib.range 1 3) (i: {
+              inbound = "sg${toString i}";
+              outbound = "direct";
+            })
+          ))
+          ++ (lib.optionals (!cfg.warp.onTrojan) [
+            {
+              inbound = "sg0";
+              outbound = "direct";
+            }
+          ]);
      };
    };
 in
 {
  options.commonSettings.proxyServer = {
    enable = mkEnableOption "sing-box as a server";
+
+    trojan = {
+      port = mkOption {
+        type = lib.types.port;
+        default = 8080;
+      };
+    };
+
+    warp = {
+      onTrojan = mkEnableOption "forward to warp in trojan";
+      onTuic = mkEnableOption "forward to warp in first two port of tuic";
+    };
  };

  config = mkIf cfg.enable {
@ -132,15 +160,10 @@ in

    networking.firewall.allowedTCPPorts = [
      80
-      8080
+      cfg.trojan.port
    ];
    networking.firewall.allowedUDPPorts = [ ] ++ (lib.range 6311 6314);

-    custom.prometheus = {
-      enable = true;
-      exporters.blackbox.enable = true;
-    };
-
    services.sing-box = {
      enable = true;
      settings = mkSingConfig {
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -4,6 +4,7 @@
    ./common-settings/autoupgrade.nix
    ./common-settings/nix-conf.nix
    ./common-settings/proxy-server.nix
+    ./disk-partitions
    ./restic.nix
    ./vaultwarden.nix
    ./prometheus
--- a/modules/nixos/disk-partitions/btrfs.nix
+++ b/modules/nixos/disk-partitions/btrfs.nix
@ -0,0 +1,46 @@
+{
+  size = "100%";
+  content = {
+    type = "btrfs";
+    extraArgs = [ "-f" ]; # Override existing partition
+    # Subvolumes must set a mountpoint in order to be mounted,
+    # unless their parent is mounted
+    subvolumes = {
+      # Subvolume name is different from mountpoint
+      "/rootfs" = {
+        mountpoint = "/";
+      };
+      # Subvolume name is the same as the mountpoint
+      "/home" = {
+        mountOptions = [ "compress=zstd" ];
+        mountpoint = "/home";
+      };
+      # Parent is not mounted so the mountpoint must be set
+      "/nix" = {
+        mountOptions = [
+          "compress=zstd"
+          "noatime"
+        ];
+        mountpoint = "/nix";
+      };
+      "/persistent" = {
+        mountOptions = [
+          "compress=zstd"
+          "noatime"
+          # Lots of dbs in /var/lib, let's disable cow
+          "nodatacow"
+        ];
+        mountpoint = "/var/lib";
+      };
+      # Subvolume for the swapfile
+      "/swap" = {
+        mountpoint = "/.swapvol";
+        swap = {
+          swapfile.size = "2G";
+        };
+      };
+    };
+
+    mountpoint = "/partition-root";
+  };
+}
--- a/modules/nixos/disk-partitions/default.nix
+++ b/modules/nixos/disk-partitions/default.nix
@ -0,0 +1,15 @@
+{ lib, ... }:
+{
+  options = {
+    diskPartitions = lib.mkOption {
+      type = lib.types.attrs;
+      default = { };
+    };
+  };
+  config = {
+    diskPartitions = {
+      btrfs = import ./btrfs.nix;
+      grubMbr = import ./grub-mbr.nix;
+    };
+  };
+}
--- a/modules/nixos/disk-partitions/grub-mbr.nix
+++ b/modules/nixos/disk-partitions/grub-mbr.nix
@ -0,0 +1,4 @@
+{
+  size = "1M";
+  type = "EF02"; # for grub MBR
+}
--- a/modules/nixos/prometheus/blackbox.nix
+++ b/modules/nixos/prometheus/blackbox.nix
@ -1,93 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.blackbox.enable) {
-    services.prometheus.exporters.blackbox = {
-      enable = true;
-      listenAddress = "127.0.0.1";
-      configFile = pkgs.writeText "blackbox.config.yaml" (
-        lib.generators.toYAML { } {
-          modules = {
-            tcp4_connect = {
-              prober = "tcp";
-              tcp = {
-                ip_protocol_fallback = false;
-                preferred_ip_protocol = "ip4";
-                tls = false;
-              };
-              timeout = "15s";
-            };
-          };
-        }
-      );
-    };
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "blackbox";
-        scrape_interval = "1m";
-        metrics_path = "/probe";
-        params = {
-          module = [ "tcp4_connect" ];
-        };
-        static_configs = [
-          {
-            targets = [
-              "tok-00.namely.icu:8080"
-              "la-00.video.namely.icu:8080"
-              "auth.xinyang.life:443"
-              "home.xinyang.life:8000"
-            ];
-          }
-        ];
-        relabel_configs = [
-          {
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox_exporter";
-        static_configs = [
-          { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }
-        ];
-      }
-    ];
-
-    custom.prometheus.ruleModules = [
-      {
-        name = "probe_alerts";
-        rules = [
-          {
-            alert = "HighProbeLatency";
-            expr = "probe_duration_seconds > 0.5";
-            for = "2m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High request latency on {{ $labels.instance }}";
-              description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes.";
-            };
-          }
-        ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/caddy.nix
+++ b/modules/nixos/prometheus/caddy.nix
@ -1,40 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.caddy.enable) {
-    services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable ''
-      servers {
-        metrics
-      }
-    '';
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "caddy";
-        static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ];
-      }
-    ];
-
-    custom.prometheus.ruleModules = [
-      {
-        name = "caddy_alerts";
-        rules = [
-          {
-            alert = "UpstreamHealthy";
-            expr = "caddy_reverse_proxy_upstreams_healthy != 1";
-            for = "5m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Upstream {{ $labels.unstream }} not healthy";
-            };
-          }
-        ];
-      }
-    ];
-  };
-
-}
--- a/modules/nixos/prometheus/default.nix
+++ b/modules/nixos/prometheus/default.nix
@ -1,21 +1,16 @@
 {
  config,
-  pkgs,
  lib,
  ...
 }:
-
-with lib;
-
 let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    mkIf
+    types
+    ;
  cfg = config.custom.prometheus;
-  mkExporterOption =
-    enableOption:
-    (mkOption {
-      type = types.bool;
-      default = enableOption;
-      description = "Enable this exporter";
-    });

  mkRulesOption = mkOption {
    type = types.listOf (
@ -30,38 +25,36 @@ let
 in
 {
  imports = [
-    ./blackbox.nix
-    ./caddy.nix
-    ./gotosocial.nix
-    ./immich.nix
-    ./miniflux.nix
-    ./ntfy-sh.nix
-    ./restic.nix
+    ./exporters.nix
+    ./grafana.nix
  ];

  options = {
+    custom.monitoring = {
+      grafana = {
+        enable = mkEnableOption "grafana with oauth only";
+      };
+    };
    custom.prometheus = {
      enable = mkEnableOption "Prometheus instance";
-      exporters = {
-        enable = mkOption {
-          type = types.bool;
-          default = false;
-          description = "Enable Prometheus exporter on every supported services";
-        };
-
-        restic.enable = mkExporterOption config.services.restic.server.enable;
-        blackbox.enable = mkExporterOption false;
-        caddy.enable = mkExporterOption config.services.caddy.enable;
-        gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
-        immich.enable = mkExporterOption config.services.immich.enable;
-        miniflux.enable = mkExporterOption config.services.miniflux.enable;
-        ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable;
-      };
-      grafana = {
-        enable = mkEnableOption "Grafana Cloud";
-        password_file = mkOption { type = types.path; };
-      };
      ruleModules = mkRulesOption;
+      exporters = {
+        enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
+        node = {
+          enable = mkEnableOption "node exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+        blackbox = {
+          enable = mkEnableOption "blackbox exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+      };
    };
  };

@ -78,46 +71,18 @@ in
        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
      '';
    };
-
    services.prometheus = mkIf cfg.enable {
      enable = true;
      port = 9091;
      globalConfig.external_labels = {
        hostname = config.networking.hostName;
      };
-      remoteWrite = mkIf cfg.grafana.enable [
-        {
-          name = "grafana";
-          url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
-          basic_auth = {
-            username = "1340065";
-            password_file = cfg.grafana.password_file;
-          };
-        }
-      ];
-      exporters = {
-        node = {
-          enable = true;
-          enabledCollectors = [
-            "loadavg"
-            "time"
-            "systemd"
-          ];
-          listenAddress = "127.0.0.1";
-          port = 9100;
-        };
-      };
+
      scrapeConfigs = [
        {
          job_name = "prometheus";
          static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
        }
-        {
-          job_name = "node";
-          static_configs = [
-            { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
-          ];
-        }
      ];

      alertmanager = {
@ -163,55 +128,8 @@ in
    };
    custom.prometheus.ruleModules = [
      {
-        name = "system_alerts";
+        name = "prometheus_alerts";
        rules = [
-          {
-            alert = "SystemdFailedUnits";
-            expr = "node_systemd_unit_state{state=\"failed\"} > 0";
-            for = "5m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Systemd has failed units on {{ $labels.instance }}";
-              description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
-            };
-          }
-          {
-            alert = "HighLoadAverage";
-            expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
-            for = "1m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High load average detected on {{ $labels.instance }}";
-              description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
-            };
-          }
-          {
-            alert = "HighTransmitTraffic";
-            expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
-            for = "1m";
-            labels = {
-              severity = "warning";
-            };
-            annotations = {
-              summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
-              description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
-            };
-          }
-          {
-            alert = "NetworkTrafficExceedLimit";
-            expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
-            for = "0m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Outbound network traffic exceed 300GB for last 30 day";
-            };
-          }
          {
            alert = "JobDown";
            expr = "up == 0";
--- a/modules/nixos/prometheus/exporters.nix
+++ b/modules/nixos/prometheus/exporters.nix
@ -0,0 +1,65 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkIf;
+  cfg = config.custom.prometheus.exporters;
+in
+{
+  config = {
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      enabledCollectors = [
+        "loadavg"
+        "time"
+        "systemd"
+      ];
+      listenAddress = cfg.node.listenAddress;
+      port = 9100;
+    };
+
+    services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
+      enable = true;
+      listenAddress = cfg.blackbox.listenAddress;
+      configFile = pkgs.writeText "blackbox.config.yaml" (
+        lib.generators.toYAML { } {
+          modules = {
+            tcp4_connect = {
+              prober = "tcp";
+              tcp = {
+                ip_protocol_fallback = false;
+                preferred_ip_protocol = "ip4";
+                tls = false;
+              };
+              timeout = "15s";
+            };
+          };
+        }
+      );
+    };
+
+    services.gotosocial.settings = {
+      metrics-enabled = true;
+    };
+
+    services.immich.environment = {
+      IMMICH_TELEMETRY_INCLUDE = "all";
+    };
+
+    services.restic.server.prometheus = true;
+    systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
+    services.ntfy-sh.settings.enable-metrics = true;
+
+    services.caddy.globalConfig = ''
+      servers {
+        metrics
+      }
+
+      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
+      }
+    '';
+  };
+}
--- a/modules/nixos/prometheus/gotosocial.nix
+++ b/modules/nixos/prometheus/gotosocial.nix
@ -1,17 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.gotosocial.enable) {
-    services.gotosocial.settings = {
-      metrics-enabled = true;
-    };
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "gotosocial";
-        static_configs = [ { targets = [ "localhost:8080" ]; } ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/grafana.nix
+++ b/modules/nixos/prometheus/grafana.nix
@ -0,0 +1,43 @@
+{ config, lib, ... }:
+let
+  cfg = config.custom.monitoring.grafana;
+in
+{
+  config = lib.mkIf cfg.enable {
+    sops.templates."grafana.env".content = ''
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${config.sops.placeholder."grafana/oauth_secret"}
+    '';
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3003;
+          root_url = "https://grafana.xinyang.life";
+          domain = "grafana.xinyang.life";
+        };
+        "auth.generic_oauth" = {
+          enabled = true;
+          name = "Kanidm";
+          client_id = "grafana";
+          scopes = "openid,profile,email,groups";
+          auth_url = "https://auth.xinyang.life/ui/oauth2";
+          token_url = "https://auth.xinyang.life/oauth2/token";
+          api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
+          use_pkce = true;
+          use_refresh_token = true;
+          allow_sign_up = true;
+          login_attribute_path = "preferred_username";
+          groups_attribute_path = "groups";
+          role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
+          allow_assign_grafana_admin = true;
+          auto_login = true;
+        };
+        "auth" = {
+          disable_login_form = true;
+        };
+      };
+    };
+    systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.templates."grafana.env".path;
+  };
+}
--- a/modules/nixos/prometheus/immich.nix
+++ b/modules/nixos/prometheus/immich.nix
@ -1,25 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-  immichEnv = config.services.immich.environment;
-  metricPort =
-    if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv then
-      immichEnv.IMMICH_API_METRICS_PORT
-    else
-      8081;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) {
-    services.immich.environment = {
-      IMMICH_METRICS = "true";
-    };
-
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "immich";
-        static_configs = [ { targets = [ "127.0.0.1:${toString metricPort}" ]; } ];
-      }
-    ];
-  };
-
-}
--- a/modules/nixos/prometheus/miniflux.nix
+++ b/modules/nixos/prometheus/miniflux.nix
@ -1,15 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.miniflux.enable) {
-    systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "miniflux";
-        static_configs = [ { targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; } ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/ntfy-sh.nix
+++ b/modules/nixos/prometheus/ntfy-sh.nix
@ -1,15 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.custom.prometheus;
-in
-{
-  config = lib.mkIf (cfg.enable && cfg.exporters.ntfy-sh.enable) {
-    services.ntfy-sh.settings.enable-metrics = true;
-    services.prometheus.scrapeConfigs = [
-      {
-        job_name = "ntfy-sh";
-        static_configs = [ { targets = [ "ntfy.xinyang.life" ]; } ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/restic.nix
+++ b/modules/nixos/prometheus/restic.nix
@ -3,17 +3,26 @@ let
  cfg = config.custom.prometheus;
 in
 {
-  config = lib.mkIf (cfg.enable && cfg.exporters.restic.enable) {
+  config = {
    services.restic.server.prometheus = true;

-    services.prometheus.scrapeConfigs = [
-      (lib.mkIf cfg.exporters.restic.enable {
-        job_name = "restic";
-        static_configs = [ { targets = [ config.services.restic.server.listenAddress ]; } ];
-      })
-    ];
+    custom.prometheus.templates.scrape.mkResticScrapes =
+      {
+        address,
+        port ? null,
+        ...
+      }:
+      let
+        portStr = if port then ":${toString port}" else "";
+      in
+      [
+        (lib.mkIf cfg.exporters.restic.enable {
+          job_name = "restic";
+          static_configs = [ { targets = [ "${address}${portStr}" ]; } ];
+        })
+      ];

-    custom.prometheus.ruleModules = [
+    custom.prometheus.templates.rules.mkResticRules = [
      {
        name = "restic_alerts";
        rules = [
--- a/modules/nixos/restic.nix
+++ b/modules/nixos/restic.nix
@ -6,63 +6,150 @@
  ...
 }:
 let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    mkDefault
+    mkIf
+    types
+    getExe
+    ;
  cfg = config.custom.restic;
+  mapBtrfsRoots =
+    rootDir:
+    let
+      backupDir = lib.removeSuffix "/" "/backup${rootDir}";
+      slash = if rootDir == "/" then "" else "/";
+      awk = getExe pkgs.gawk;
+      continueIfInExclude = ''
+        exclude_subv="${toString cfg.btrfsExcludeSubvolume}"
+        found=false
+        for subv in $exclude_subv; do
+            if [[ "$subvol" == "$subv" ]]; then
+                found=true
+                echo "$subvol is in exclude subvolumes, skipped"
+                break
+            fi
+        done
+        $found && continue
+      '';
+    in
+    {
+      backupPrepareCommand = ''
+        echo "Creating snapshot for ${rootDir}"
+        subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -o "${rootDir}" | ${awk} '{print $NF}')
+        mkdir -p "${backupDir}"
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}" "${backupDir}/rootfs"
+        for subvol in $subvolumes; do
+            ${continueIfInExclude}
+            [[ /"$subvol" == "${backupDir}"* ]] && continue
+
+            snapshot_path=$(dirname "${backupDir}/$subvol")
+            mkdir -p "$snapshot_path"
+
+            echo "Creating snapshot for subvolume: $subvol at $snapshot_path"
+            ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}${slash}$subvol" "$snapshot_path"
+        done
+      '';
+
+      # Note that all the manually created snapshots under backupDir will also be cleaned
+      backupCleanupCommand = ''
+        # Only find snapshots under backup directory
+        subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -s -o "${backupDir}" | ${awk} '{print $NF}')
+        for subvol in $subvolumes; do
+            echo "Removing snapshot for subvolume: $subvol"
+            ${pkgs.btrfs-progs}/bin/btrfs subvolume delete "$subvol"
+        done
+      '';
+    };
+
+  btrfsFs = lib.attrsets.filterAttrs (
+    n: v: v.fsType == "btrfs" && ((isNull cfg.btrfsRoots) || (builtins.elem n cfg.btrfsRoots))
+  ) config.fileSystems;
+  btrfsFsRoot = builtins.attrNames btrfsFs;
+  btrfsCommands = (map mapBtrfsRoots btrfsFsRoot);
 in
 {
  options = {
    custom.restic = {
-      enable = lib.mkEnableOption "restic";
-      paths = lib.mkOption {
-        type = lib.types.listOf lib.types.str;
+      enable = mkEnableOption "restic";
+      paths = mkOption {
+        type = types.listOf types.str;
        default = [
          "/home"
          "/var/lib"
        ];
      };
-      prune = lib.mkEnableOption "auto prune remote restic repo";
-      repositoryFile = lib.mkOption {
-        type = lib.types.str;
-        default = "";
+      prune = mkEnableOption "auto prune remote restic repo";
+      btrfsRoots = mkOption {
+        type = types.nullOr (types.listOf types.str);
+        default = [ "/" ];
+        description = ''
+          Includeded btrfs roots. `null` means snapshot all btrfs filesystems under config.fileSystems.
+        '';
      };
-      passwordFile = lib.mkOption {
-        type = lib.types.str;
-        default = "";
+      btrfsExcludeSubvolume = mkOption {
+        type = types.listOf types.str;
+        default = [
+          "nix"
+          "rootfs"
+          "swap"
+          "var/tmp"
+        ];
+        example = lib.literalExpression ''
+          [ "var/tmp" "srv" ]
+        '';
+      };
+      backupPrepareCommand = mkOption {
+        type = types.listOf types.str;
+      };
+      backupCleanupCommand = mkOption {
+        type = types.listOf types.str;
      };
    };
  };
-  config = lib.mkIf cfg.enable {
-    services.restic.backups.${config.networking.hostName} = lib.mkMerge [
-      {
-        repositoryFile = cfg.repositoryFile;
-        passwordFile = cfg.passwordFile;
-        exclude = [
-          "/home/*/.cache"
-          "/home/*/.cargo"
-          "/home/*/.local/share/Steam"
-          "/home/*/.local/share/flatpak"
-        ];
-        timerConfig = {
-          OnCalendar = "00:05";
-          RandomizedDelaySec = "5h";
-        };
-        pruneOpts = lib.mkIf cfg.prune [
-          "--keep-daily 7"
-          "--keep-weekly 5"
-          "--keep-monthly 12"
-          "--keep-yearly 75"
-        ];
-        paths = lib.mkDefault cfg.paths;
-        initialize = true;
-      }
-      (lib.mkIf (config.fileSystems."/".fsType == "btrfs") {
-        backupPrepareCommand = ''
-          ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r / backup
-        '';
-        backupCleanupCommand = ''
-          ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup 
-        '';
-        paths = map (p: "/backup" + p) cfg.paths;
-      })
-    ];
+  config = mkIf cfg.enable {
+    services.restic.backups.${config.networking.hostName} = {
+      repositoryFile = config.sops.secrets."restic/repo_url".path;
+      passwordFile = config.sops.secrets."restic/repo_password".path;
+      exclude = [
+        "**/.cache"
+        "**/.local/share/Steam"
+        "**/.local/share/flatpak"
+
+        "**/.cargo"
+        "**/.rustup"
+
+        "**/node_modules"
+
+        "*.pyc"
+        "*.pyo"
+        "**/__pycache__"
+        "**/.virtualenvs"
+        "**/.venv"
+
+        # temp files / lock files
+        "*.sqlite-wal"
+        "*.sqlite-shm"
+        "*.db-wal"
+        "*.db-shm"
+      ];
+      timerConfig = {
+        OnCalendar = "00:05";
+        RandomizedDelaySec = "5h";
+      };
+      pruneOpts = mkIf cfg.prune [
+        "--keep-daily 7"
+        "--keep-weekly 5"
+        "--keep-monthly 12"
+        "--keep-yearly 75"
+      ];
+      paths = mkDefault cfg.paths;
+      initialize = true;
+      backupPrepareCommand = lib.strings.concatLines cfg.backupPrepareCommand;
+      backupCleanupCommand = lib.strings.concatLines cfg.backupCleanupCommand;
+    };
+    custom.restic.backupPrepareCommand = map (x: x.backupPrepareCommand) btrfsCommands;
+    custom.restic.backupCleanupCommand = map (x: x.backupCleanupCommand) btrfsCommands;
  };
 }
--- a/overlays/add-pkgs.nix
+++ b/overlays/add-pkgs.nix
@ -1,5 +1,12 @@
-(
-  final: prev:
-  {
-  }
-)
+(final: prev: {
+  ubootOrangePiR1LtsPackage = prev.buildUBoot {
+    defconfig = "orangepi-r1-plus-lts-rk3328_defconfig";
+    enableParallelBuilding = true;
+
+    BL31 = "${prev.armTrustedFirmwareRK3328}/bl31.elf";
+    filesToInstall = [
+      "u-boot.itb"
+      "idbloader.img"
+    ];
+  };
+})
--- a/overlays/my-lib/default.nix
+++ b/overlays/my-lib/default.nix
@ -0,0 +1,3 @@
+{
+}
+// (import ./prometheus.nix)
--- a/overlays/my-lib/prometheus.nix
+++ b/overlays/my-lib/prometheus.nix
@ -0,0 +1,199 @@
+let
+  mkFunction = f: (targets: (map f targets));
+  mkPort = port: if isNull port then "" else ":${toString port}";
+in
+{
+  mkScrapes = mkFunction (
+    {
+      name,
+      address,
+      port ? 443,
+      scheme ? "https",
+      ...
+    }:
+    {
+      job_name = "${name}(${address})";
+      scheme = scheme;
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+  );
+
+  mkCaddyScrapes = mkFunction (
+    {
+      address,
+      port ? 2019,
+      ...
+    }:
+    {
+      job_name = "caddy_${address}";
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+  );
+
+  mkCaddyRules = mkFunction (
+    {
+      host ? "",
+      name ? "caddy_alerts_${host}",
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "UpstreamHealthy";
+          expr = "caddy_reverse_proxy_upstreams_healthy != 1";
+          for = "5m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Upstream {{ $labels.unstream }} not healthy";
+          };
+        }
+      ];
+    }
+  );
+
+  mkNodeScrapes = mkFunction (
+    {
+      address,
+      port ? 9100,
+      ...
+    }:
+    {
+      job_name = "node_${address}";
+      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
+    }
+  );
+
+  mkNodeRules = mkFunction (
+    {
+      host ? "",
+      name ? "system_alerts_${host}",
+      ...
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "SystemdFailedUnits";
+          expr = "node_systemd_unit_state{state=\"failed\"} > 0";
+          for = "5m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Systemd has failed units on {{ $labels.instance }}";
+            description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
+          };
+        }
+        {
+          alert = "HighLoadAverage";
+          expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
+          for = "1m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High load average detected on {{ $labels.instance }}";
+            description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
+          };
+        }
+        {
+          alert = "HighTransmitTraffic";
+          expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
+          for = "1m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
+            description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
+          };
+        }
+        {
+          alert = "NetworkTrafficExceedLimit";
+          expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
+          for = "0m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "Outbound network traffic exceed 300GB for last 30 day";
+          };
+        }
+      ];
+    }
+  );
+
+  mkBlackboxScrapes = mkFunction (
+    {
+      hostAddress,
+      hostPort ? 9115,
+      targetAddresses,
+      ...
+    }:
+    {
+      job_name = "blackbox(${hostAddress})";
+      scrape_interval = "1m";
+      metrics_path = "/probe";
+      params = {
+        module = [ "tcp4_connect" ];
+      };
+      static_configs = [
+        {
+          targets = targetAddresses;
+        }
+      ];
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          source_labels = [ "__param_target" ];
+          target_label = "instance";
+        }
+        {
+          target_label = "__address__";
+          replacement = "${hostAddress}${mkPort hostPort}";
+        }
+      ];
+    }
+  );
+
+  mkBlackboxRules = mkFunction (
+    {
+      host ? "",
+      name ? "probe_alerts_${host}",
+    }:
+    {
+      inherit name;
+      rules = [
+        {
+          alert = "HighProbeLatency";
+          expr = "probe_duration_seconds > 0.5";
+          for = "3m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High request latency on {{ $labels.instance }}";
+            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+          };
+        }
+        {
+          alert = "VeryHighProbeLatency";
+          expr = "probe_duration_seconds > 1";
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "High request latency on {{ $labels.instance }}";
+            description = "Request latency is above 0.5 seconds for the last 3 minutes.";
+          };
+        }
+      ];
+    }
+  );
+}
--- a/scripts/nixos-updater.py
+++ b/scripts/nixos-updater.py
@ -0,0 +1,90 @@
+import requests
+import os
+import socket
+import json
+from os import path as osp
+from dataclasses import dataclass
+
+"""
+This updater consists of several parts: 
+
+- Update checker: Check an url for update (if outPath is different from /run/current-system or some specified profile) or maybe use timestamp for update
+- Nix copy --from: copy from remote. Need to specify remote url.
+- Create a symlink: /run/next-system -> <new system derivation>
+- Listen for POST request to trigger system switch (optional)
+"""
+
+
+@dataclass
+class GarnixConfig:
+    token: str
+
+
+@dataclass
+class Config:
+    check_type: str
+    check_url: str
+    remote_url: str
+    garnix: GarnixConfig
+    hostname: str = socket.gethostname()
+
+
+class Nix:
+    def __init__(self, args):
+        self.args = args
+
+    def copy_from_remote(self):
+        # run nix copy with subprocess
+        pass
+
+    def eval(self):
+
+
+class Updater:
+    def __init__(self, config: Config):
+        self.config = config
+
+        # TODO: Make this configurable
+        self.current_drv = os.readlink("/run/current-system")
+        self.next_dev = None
+
+    # checkers take an url and returns the outPath of the latest success build
+    def garnix_checker(self) -> str:
+        domain = "garnix.io"
+        build_endpoint = "/api/build/commit"
+
+        # Latest commit from git
+
+        # Check build status of this commit
+        resp = requests.get(
+            f"https://{domain}{build_endpoint}/40b1e9ff23aaa5f555420dd22414c3f137a02cfe"
+        )
+        # Raise error if status code is not valid
+
+        # Fetch outPath from eval endpoint
+        # TODO: In theory, this could be done by parsing raw log from garnix.
+
+        # Try to evaluate locally if eval endpoint is not configured
+
+        resp = resp.json()
+        # TODO
+        return "null"
+
+    def hydra_checker(self) -> str:
+        # TODO
+        return "null"
+
+    # Check for update
+    def poll(self) -> str | None:
+        cfg = self.config
+        if cfg.check_type == "garnix":
+            pass
+        elif cfg.check_type == "hydra":
+            pass
+        else:
+            pass
+        pass
+
+
+if __name__ == "__main__":
+    pass
Author	SHA1	Message	Date
xinyangli	ec63c8b6cc	modules/prometheus: refactor	2024-11-30 11:25:35 +08:00
xinyangli	ebf69d94dd	modules/restic: snapshot all btrfs subvolumes	2024-11-28 12:07:22 +08:00
xinyangli	2327a171b8	biotite,thorite: added	2024-11-27 17:48:58 +08:00
xinyangli	3dc3775a6c	modules: add disko partition module for common partition patterns	2024-11-27 17:43:56 +08:00
xinyangli	0f4a315658	calcite: move input methods from hm to system config	2024-11-26 17:37:16 +08:00
xinyangli	1c40bbc98f	dolomite: drop tok-00, add fre-00	2024-11-26 17:36:16 +08:00
xinyangli	5b19d8a97e	weilite: more media services	2024-11-25 01:27:03 +08:00
xinyangli	7c5c8be995	dolomite: disable warp	2024-11-24 23:43:32 +08:00
xinyangli	ca8f27bafa	osmium: added	2024-11-24 21:58:43 +08:00
xinyangli	02636ac5a1	dolomite: fix hk-00	2024-11-24 21:44:45 +08:00