modules/restic: snapshot all btrfs subvolumes

machines/calcite/configuration.nix

@@ -182,6 +182,17 @@ in
           };
         };
       };
+      "keydous" = {
+        ids = [
+          "25a7:fa14"
+          "3151:4002"
+        ];
+        settings = {
+          main = {
+            capslock = "overload(control, esc)";
+          };
+        };
+      };
     };
   };
 
@@ -324,26 +335,27 @@ in
   '';
 
   sops.secrets = {
-    restic_repo_calcite_password = {
+    "restic/repo_url" = {
       owner = "xin";
       sopsFile = ./secrets.yaml;
     };
-    restic_repo_calcite = {
+    "restic/repo_password" = {
       owner = "xin";
       sopsFile = ./secrets.yaml;
     };
-    sing_box_url = {
-      owner = "root";
-      sopsFile = ./secrets.yaml;
-    };
     "gitea/envfile" = {
       owner = "root";
       sopsFile = ./secrets.yaml;
     };
   };
-  custom.restic.enable = true;
+
-  custom.restic.repositoryFile = config.sops.secrets.restic_repo_calcite.path;
+  custom.restic = {
-  custom.restic.passwordFile = config.sops.secrets.restic_repo_calcite_password.path;
+    enable = true;
+    paths = [
+      "/backup/rootfs/var/lib"
+      "/backup/home"
+    ];
+  };
 
   custom.forgejo-actions-runner = {
     enable = false;

machines/calcite/secrets.yaml

@@ -1,6 +1,6 @@
-restic_repo_calcite_password: ENC[AES256_GCM,data:9ALTQULAMyLY4FIxuVztf9r3,iv:fObBBeqpHAVYl8YUopz9fZd3YWB+0sc8l+sR12rmxb4=,tag:l3xDc2/cpQr38X/cd7qMXA==,type:str]
+restic:
-restic_repo_calcite: ENC[AES256_GCM,data:ELvSvoBfulbsoMvRMt2bVo9KiNQAuHomblZcAwJ+g0tHELkq65kaaGwMsNy1AttBfiD7RrQsKifX/YTUGmuz1mDg0WqkV/Mv,iv:HKz96YgVahxh+t3AEqe09mTE01uT+VrUYt04H6zyS9g=,tag:llFeeN7ryTZI9gLlYIRhCg==,type:str]
+    repo_url: ENC[AES256_GCM,data:x/g1nZQ59SavVG+u5apNmBQ0Y5uQ9N0EKVh6qovqeP/Z7tmkudJtlBFD35C0ZidcQLAqTaZk1FFh8Ikjo4OcQSdTsx9BGvT4,iv:RQMOSEacDHXjYceBaAW4sFGk38vkijHuADcTS3DMxa8=,tag:769rLA2eRKjDrAaL/jERbA==,type:str]
-sing_box_url: ENC[AES256_GCM,data:2z2bDKdn51o1eaqhgE0pTg4FWcO8wcLNlnBZ69Q3Jm5GCxkXxsxN7DgqQvRVeakOHvaenQotF+nc6tlhKPsyzdQeG0yl3YYhGb9o3DkmpUjC6lalMSoiw1rSMVyBg4KYCWxmhR9iRurun62+5INGZwwHVqAjgWJhy/9+pdIFtgKyd/t0JhSU,iv:gIGbvRd88vZu3cVW7e4emZmmNO8QcubLrxS1sCwi4Co=,tag:AzLLtcA9jAbeuo6eWU6ilw==,type:str]
+    repo_password: ENC[AES256_GCM,data:jqsIP1R5/yX8F0oYaSXACx6C,iv:KckzqctKLnmay+d30/Y4IttiASxYnMw6IHQrtwP2YdQ=,tag:L/Ij51UU1om48I8fd4iuwA==,type:str]
 gitea:
     envfile: ENC[AES256_GCM,data:CK+JNELuzjKgWnImuV4Euif3f3nNOACOrvc4NiIXs+q/F7QWrtpb3TK8/FrLNQk=,iv:QSDrlKJCBld2gDx/y1sT8anh37GhqSS2QZd2JJi5Yis=,tag:x5T6h59LBXhEyVwSr2dnuQ==,type:str]
 sops:
@@ -27,8 +27,8 @@ sops:
             WGlLdXVoZlp3bEFXZjlMdG1VOUZDNUkKQ2NNTE3OsNUr2pOI7qeNFSCVkUIVRS+g
             FG5FbJJcFihXqr+Qo0nZkq+xq07vIia7mKoqyoIfkKwweiVzDKyrkQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-09T06:41:02Z"
+    lastmodified: "2024-11-28T03:55:19Z"
-    mac: ENC[AES256_GCM,data:Hf8QYvRWxfs/JDOIAVnX5M0kv9Ktncfzq+Zf7i32TTsa94ShrgbUYVxQbRviOFDbjLfzswGKikLQ2EHLlH1KOFs7+mKKz5PKVAWJZnkAPa2oFXs41BcXLIg8sf4dhFxjzzhakeUX9Q0z4evJ1vMX06/VnnpHVSMhsnenSfBhWIA=,iv:uXKf2oYSb+0IWp6Ch0XuoFUIaUBiAW7Z8R9Z7LSdLvY=,tag:0VAcFakwCrHGZW5I8jmydA==,type:str]
+    mac: ENC[AES256_GCM,data:VH7RnRT33ltsxycuSsUsM+64onQeClwQ3fIHUVQUyRJ6t7aJkBiGMQ80QtmwGE5CJTbq7LV4cis5Pq/f9vTb0SsY4tCSIgXNAE2zW2rjjQKjdHr+rnnKSJExJA+k2tL06Q/FUu+3SP7pVSaYBGQKb53UAbHsdJYbx00Ko6MzZ7U=,iv:EiYhbr6o4n3kGEEWKXeWmDPSb5hOvUhRH7N2ZLPRHmQ=,tag:BdI140bhvBW0bwQPpRYiRw==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.1

machines/massicot/secrets.yaml

@@ -7,8 +7,8 @@ miniflux:
 forgejo:
     env: ENC[AES256_GCM,data:TMeguXfanISeyvsay9SBqm3SSGKpp5nCkqhHblf0QHNzHWGQKwpORmWfOtVfgOh9qdDqq8wYBpXznmbvixjV,iv:IR/rMoAIvZCw9FURmau4+g8c3pvI9BRs7v1NJ5ia4jI=,tag:kjwf6RN5HN8I2sUhDcr4UQ==,type:str]
 restic:
-    repo: ENC[AES256_GCM,data:/vybkTU7LMWSlco9W2pJouU9wm4okXClSHXQMCA6SGIHWp4Ppl6C+jS4sNJALc6ntKzcEHyWO/R3JPjQKjZNH4YtrnNQp/ZY9g==,iv:gAvp6blg5JuBKzLw6YSgM1Uc24Aesov3ttCRXZXBvJw=,tag:pvH1y6BFOl7jIn/qQejUbQ==,type:str]
+    repo_url: ENC[AES256_GCM,data:GMHbrjgwajnYSiqtoYaKiFT/aDWDwlzEkvMLPzYf7C9PvLr7T4zeWyAA9//8huldyxO3+nk6O9lR9ORZKZfb8/MYB7nRB03sZQ==,iv:6uBhsksOGDjoc13U2xWLz7I+0fzGRhnw0nStACqlnug=,tag:uhH28NYq+ly1bmCV/cpxkQ==,type:str]
-    password: ENC[AES256_GCM,data:5eIIBtGtBFwcAQ+ZwTYOtg==,iv:3GEM8Imu0i1aTwwSspvz2EzwJOXUC/b15hzkFFuZ+YY=,tag:wscba+nMtshldgUtcEKnOw==,type:str]
+    repo_password: ENC[AES256_GCM,data:jRHNgOk5ChWdqMKsd/V4Xg==,iv:wrgF5pau/RylG1nmJYmvrZ02o67qkkT5PrZAQlXb6Qo=,tag:X0WVpMqi8xeoATss/sSPMA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -33,8 +33,8 @@ sops:
             dnFBa0lDWWZtS1BHdzBoVzNTaGNkSEEKi/W1n7RT8NpTp00SBMwxsUJAPDhumJ/i
             V2VnaSNwouD3SswTcoBzqQpBP9XrqzjIYGke90ZODFQbMY9WDQ+O0g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-30T07:19:35Z"
+    lastmodified: "2024-11-28T03:57:35Z"
-    mac: ENC[AES256_GCM,data:WSGvA1RkChrD07Sf4BFVMbdTXQYxAHeGGQ52e+pnPh0lZPOzMc9sLDrBPqDK2OfrHC+hK8RC7FxQTGs6G/oBB4nUzIZPn9WycTiU5elwWDfktizH0gr3EJDm7Gs+bTWQpwdoJZGZ8XErK+yegCaKL5cSOSTlBBbQOnZfnoNBg5c=,iv:xyJRFfxHC2xV0ro4CbdOPau1zORxA64OqpvKr4aFZvQ=,tag:c9NA90d5WTK2pfxwoyOX5A==,type:str]
+    mac: ENC[AES256_GCM,data:xjZrlwfWLtZNYfH+KiE2ICt9Jo4nx/LKaEYi/ECN/Od+ZTjety0V6RJ/RfmI6q3K1WMj0sAGc56hCZ0iOn25L8wK6dc14hZVoSwwbIiQ7hTQE5LcK+NbXNmy3r/YC855DHG9kE08eYGHdNcBbckZg3HhkHQ9UYS/Ox/QFFuBa5Q=,iv:N3AW+sr9ET3c/ArXr176haRewYFsfgsNn+hkC0MDJwA=,tag:SCikn+F8btuSBswV+oCdXg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
-    version: 3.9.0
+    version: 3.9.1

machines/massicot/services/restic.nix

@@ -12,22 +12,26 @@ let
 in
 {
   sops.secrets = {
-    "restic/repo" = {
+    "restic/repo_url" = {
       sopsFile = ../secrets.yaml;
     };
-    "restic/password" = {
+    "restic/repo_password" = {
       sopsFile = ../secrets.yaml;
     };
   };
 
   custom.restic = {
     enable = true;
-    repositoryFile = config.sops.secrets."restic/repo".path;
-    passwordFile = config.sops.secrets."restic/password".path;
     paths = [
       "/backup"
       "/mnt/storage"
     ];
+    backupPrepareCommand = [
+      (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite")
+      (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3")
+      (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite")
+      (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db")
+    ];
   };
 
   services.postgresqlBackup = {
@@ -38,12 +42,6 @@ in
   };
 
   services.restic.backups.${config.networking.hostName} = {
-    backupPrepareCommand = builtins.concatStringsSep "\n" [
-      (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite")
-      (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3")
-      (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite")
-      (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db")
-    ];
     extraBackupArgs = [
       "--limit-upload=1024"
     ];

machines/minimal/default.nix

@@ -0,0 +1,25 @@
+{
+  lib,
+  ...
+}:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+  ];
+
+  boot.initrd.availableKernelModules =
+    [
+    ];
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

modules/nixos/default.nix

@@ -4,6 +4,7 @@
     ./common-settings/autoupgrade.nix
     ./common-settings/nix-conf.nix
     ./common-settings/proxy-server.nix
+    ./disk-partitions
     ./restic.nix
     ./vaultwarden.nix
     ./prometheus

modules/nixos/restic.nix

@@ -6,63 +6,150 @@
   ...
 }:
 let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    mkDefault
+    mkIf
+    types
+    getExe
+    ;
   cfg = config.custom.restic;
+  mapBtrfsRoots =
+    rootDir:
+    let
+      backupDir = lib.removeSuffix "/" "/backup${rootDir}";
+      slash = if rootDir == "/" then "" else "/";
+      awk = getExe pkgs.gawk;
+      continueIfInExclude = ''
+        exclude_subv="${toString cfg.btrfsExcludeSubvolume}"
+        found=false
+        for subv in $exclude_subv; do
+            if [[ "$subvol" == "$subv" ]]; then
+                found=true
+                echo "$subvol is in exclude subvolumes, skipped"
+                break
+            fi
+        done
+        $found && continue
+      '';
+    in
+    {
+      backupPrepareCommand = ''
+        echo "Creating snapshot for ${rootDir}"
+        subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -o "${rootDir}" | ${awk} '{print $NF}')
+        mkdir -p "${backupDir}"
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}" "${backupDir}/rootfs"
+        for subvol in $subvolumes; do
+            ${continueIfInExclude}
+            [[ /"$subvol" == "${backupDir}"* ]] && continue
+
+            snapshot_path=$(dirname "${backupDir}/$subvol")
+            mkdir -p "$snapshot_path"
+
+            echo "Creating snapshot for subvolume: $subvol at $snapshot_path"
+            ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}${slash}$subvol" "$snapshot_path"
+        done
+      '';
+
+      # Note that all the manually created snapshots under backupDir will also be cleaned
+      backupCleanupCommand = ''
+        # Only find snapshots under backup directory
+        subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -s -o "${backupDir}" | ${awk} '{print $NF}')
+        for subvol in $subvolumes; do
+            echo "Removing snapshot for subvolume: $subvol"
+            ${pkgs.btrfs-progs}/bin/btrfs subvolume delete "$subvol"
+        done
+      '';
+    };
+
+  btrfsFs = lib.attrsets.filterAttrs (
+    n: v: v.fsType == "btrfs" && ((isNull cfg.btrfsRoots) || (builtins.elem n cfg.btrfsRoots))
+  ) config.fileSystems;
+  btrfsFsRoot = builtins.attrNames btrfsFs;
+  btrfsCommands = (map mapBtrfsRoots btrfsFsRoot);
 in
 {
   options = {
     custom.restic = {
-      enable = lib.mkEnableOption "restic";
+      enable = mkEnableOption "restic";
-      paths = lib.mkOption {
+      paths = mkOption {
-        type = lib.types.listOf lib.types.str;
+        type = types.listOf types.str;
         default = [
           "/home"
           "/var/lib"
         ];
       };
-      prune = lib.mkEnableOption "auto prune remote restic repo";
+      prune = mkEnableOption "auto prune remote restic repo";
-      repositoryFile = lib.mkOption {
+      btrfsRoots = mkOption {
-        type = lib.types.str;
+        type = types.nullOr (types.listOf types.str);
-        default = "";
+        default = [ "/" ];
+        description = ''
+          Includeded btrfs roots. `null` means snapshot all btrfs filesystems under config.fileSystems.
+        '';
       };
-      passwordFile = lib.mkOption {
+      btrfsExcludeSubvolume = mkOption {
-        type = lib.types.str;
+        type = types.listOf types.str;
-        default = "";
+        default = [
+          "nix"
+          "rootfs"
+          "swap"
+          "var/tmp"
+        ];
+        example = lib.literalExpression ''
+          [ "var/tmp" "srv" ]
+        '';
+      };
+      backupPrepareCommand = mkOption {
+        type = types.listOf types.str;
+      };
+      backupCleanupCommand = mkOption {
+        type = types.listOf types.str;
       };
     };
   };
-  config = lib.mkIf cfg.enable {
+  config = mkIf cfg.enable {
-    services.restic.backups.${config.networking.hostName} = lib.mkMerge [
+    services.restic.backups.${config.networking.hostName} = {
-      {
+      repositoryFile = config.sops.secrets."restic/repo_url".path;
-        repositoryFile = cfg.repositoryFile;
+      passwordFile = config.sops.secrets."restic/repo_password".path;
-        passwordFile = cfg.passwordFile;
+      exclude = [
-        exclude = [
+        "**/.cache"
-          "/home/*/.cache"
+        "**/.local/share/Steam"
-          "/home/*/.cargo"
+        "**/.local/share/flatpak"
-          "/home/*/.local/share/Steam"
+
-          "/home/*/.local/share/flatpak"
+        "**/.cargo"
-        ];
+        "**/.rustup"
-        timerConfig = {
+
-          OnCalendar = "00:05";
+        "**/node_modules"
-          RandomizedDelaySec = "5h";
+
-        };
+        "*.pyc"
-        pruneOpts = lib.mkIf cfg.prune [
+        "*.pyo"
-          "--keep-daily 7"
+        "**/__pycache__"
-          "--keep-weekly 5"
+        "**/.virtualenvs"
-          "--keep-monthly 12"
+        "**/.venv"
-          "--keep-yearly 75"
+
-        ];
+        # temp files / lock files
-        paths = lib.mkDefault cfg.paths;
+        "*.sqlite-wal"
-        initialize = true;
+        "*.sqlite-shm"
-      }
+        "*.db-wal"
-      (lib.mkIf (config.fileSystems."/".fsType == "btrfs") {
+        "*.db-shm"
-        backupPrepareCommand = ''
+      ];
-          ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r / backup
+      timerConfig = {
-        '';
+        OnCalendar = "00:05";
-        backupCleanupCommand = ''
+        RandomizedDelaySec = "5h";
-          ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup 
+      };
-        '';
+      pruneOpts = mkIf cfg.prune [
-        paths = map (p: "/backup" + p) cfg.paths;
+        "--keep-daily 7"
-      })
+        "--keep-weekly 5"
-    ];
+        "--keep-monthly 12"
+        "--keep-yearly 75"
+      ];
+      paths = mkDefault cfg.paths;
+      initialize = true;
+      backupPrepareCommand = lib.strings.concatLines cfg.backupPrepareCommand;
+      backupCleanupCommand = lib.strings.concatLines cfg.backupCleanupCommand;
+    };
+    custom.restic.backupPrepareCommand = map (x: x.backupPrepareCommand) btrfsCommands;
+    custom.restic.backupCleanupCommand = map (x: x.backupCleanupCommand) btrfsCommands;
   };
 }