From ebf69d94dd1a5d1cd971e370acb15e7039dc600e Mon Sep 17 00:00:00 2001 From: xinyangli Date: Thu, 28 Nov 2024 12:07:16 +0800 Subject: [PATCH] modules/restic: snapshot all btrfs subvolumes --- machines/calcite/configuration.nix | 30 +++-- machines/calcite/secrets.yaml | 10 +- machines/massicot/secrets.yaml | 10 +- machines/massicot/services/restic.nix | 18 ++- machines/minimal/default.nix | 25 ++++ modules/nixos/default.nix | 1 + modules/nixos/restic.nix | 175 +++++++++++++++++++------- 7 files changed, 196 insertions(+), 73 deletions(-) create mode 100644 machines/minimal/default.nix diff --git a/machines/calcite/configuration.nix b/machines/calcite/configuration.nix index 0e2bd85..1be4864 100644 --- a/machines/calcite/configuration.nix +++ b/machines/calcite/configuration.nix @@ -182,6 +182,17 @@ in }; }; }; + "keydous" = { + ids = [ + "25a7:fa14" + "3151:4002" + ]; + settings = { + main = { + capslock = "overload(control, esc)"; + }; + }; + }; }; }; @@ -324,26 +335,27 @@ in ''; sops.secrets = { - restic_repo_calcite_password = { + "restic/repo_url" = { owner = "xin"; sopsFile = ./secrets.yaml; }; - restic_repo_calcite = { + "restic/repo_password" = { owner = "xin"; sopsFile = ./secrets.yaml; }; - sing_box_url = { - owner = "root"; - sopsFile = ./secrets.yaml; - }; "gitea/envfile" = { owner = "root"; sopsFile = ./secrets.yaml; }; }; - custom.restic.enable = true; - custom.restic.repositoryFile = config.sops.secrets.restic_repo_calcite.path; - custom.restic.passwordFile = config.sops.secrets.restic_repo_calcite_password.path; + + custom.restic = { + enable = true; + paths = [ + "/backup/rootfs/var/lib" + "/backup/home" + ]; + }; custom.forgejo-actions-runner = { enable = false; diff --git a/machines/calcite/secrets.yaml b/machines/calcite/secrets.yaml index ae33888..ee5dc17 100644 --- a/machines/calcite/secrets.yaml +++ b/machines/calcite/secrets.yaml @@ -1,6 +1,6 @@ -restic_repo_calcite_password: ENC[AES256_GCM,data:9ALTQULAMyLY4FIxuVztf9r3,iv:fObBBeqpHAVYl8YUopz9fZd3YWB+0sc8l+sR12rmxb4=,tag:l3xDc2/cpQr38X/cd7qMXA==,type:str] -restic_repo_calcite: ENC[AES256_GCM,data:ELvSvoBfulbsoMvRMt2bVo9KiNQAuHomblZcAwJ+g0tHELkq65kaaGwMsNy1AttBfiD7RrQsKifX/YTUGmuz1mDg0WqkV/Mv,iv:HKz96YgVahxh+t3AEqe09mTE01uT+VrUYt04H6zyS9g=,tag:llFeeN7ryTZI9gLlYIRhCg==,type:str] -sing_box_url: ENC[AES256_GCM,data:2z2bDKdn51o1eaqhgE0pTg4FWcO8wcLNlnBZ69Q3Jm5GCxkXxsxN7DgqQvRVeakOHvaenQotF+nc6tlhKPsyzdQeG0yl3YYhGb9o3DkmpUjC6lalMSoiw1rSMVyBg4KYCWxmhR9iRurun62+5INGZwwHVqAjgWJhy/9+pdIFtgKyd/t0JhSU,iv:gIGbvRd88vZu3cVW7e4emZmmNO8QcubLrxS1sCwi4Co=,tag:AzLLtcA9jAbeuo6eWU6ilw==,type:str] +restic: + repo_url: ENC[AES256_GCM,data:x/g1nZQ59SavVG+u5apNmBQ0Y5uQ9N0EKVh6qovqeP/Z7tmkudJtlBFD35C0ZidcQLAqTaZk1FFh8Ikjo4OcQSdTsx9BGvT4,iv:RQMOSEacDHXjYceBaAW4sFGk38vkijHuADcTS3DMxa8=,tag:769rLA2eRKjDrAaL/jERbA==,type:str] + repo_password: ENC[AES256_GCM,data:jqsIP1R5/yX8F0oYaSXACx6C,iv:KckzqctKLnmay+d30/Y4IttiASxYnMw6IHQrtwP2YdQ=,tag:L/Ij51UU1om48I8fd4iuwA==,type:str] gitea: envfile: ENC[AES256_GCM,data:CK+JNELuzjKgWnImuV4Euif3f3nNOACOrvc4NiIXs+q/F7QWrtpb3TK8/FrLNQk=,iv:QSDrlKJCBld2gDx/y1sT8anh37GhqSS2QZd2JJi5Yis=,tag:x5T6h59LBXhEyVwSr2dnuQ==,type:str] sops: @@ -27,8 +27,8 @@ sops: WGlLdXVoZlp3bEFXZjlMdG1VOUZDNUkKQ2NNTE3OsNUr2pOI7qeNFSCVkUIVRS+g FG5FbJJcFihXqr+Qo0nZkq+xq07vIia7mKoqyoIfkKwweiVzDKyrkQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-11-09T06:41:02Z" - mac: ENC[AES256_GCM,data:Hf8QYvRWxfs/JDOIAVnX5M0kv9Ktncfzq+Zf7i32TTsa94ShrgbUYVxQbRviOFDbjLfzswGKikLQ2EHLlH1KOFs7+mKKz5PKVAWJZnkAPa2oFXs41BcXLIg8sf4dhFxjzzhakeUX9Q0z4evJ1vMX06/VnnpHVSMhsnenSfBhWIA=,iv:uXKf2oYSb+0IWp6Ch0XuoFUIaUBiAW7Z8R9Z7LSdLvY=,tag:0VAcFakwCrHGZW5I8jmydA==,type:str] + lastmodified: "2024-11-28T03:55:19Z" + mac: ENC[AES256_GCM,data:VH7RnRT33ltsxycuSsUsM+64onQeClwQ3fIHUVQUyRJ6t7aJkBiGMQ80QtmwGE5CJTbq7LV4cis5Pq/f9vTb0SsY4tCSIgXNAE2zW2rjjQKjdHr+rnnKSJExJA+k2tL06Q/FUu+3SP7pVSaYBGQKb53UAbHsdJYbx00Ko6MzZ7U=,iv:EiYhbr6o4n3kGEEWKXeWmDPSb5hOvUhRH7N2ZLPRHmQ=,tag:BdI140bhvBW0bwQPpRYiRw==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1 diff --git a/machines/massicot/secrets.yaml b/machines/massicot/secrets.yaml index 0f4bbdc..9393192 100644 --- a/machines/massicot/secrets.yaml +++ b/machines/massicot/secrets.yaml @@ -7,8 +7,8 @@ miniflux: forgejo: env: ENC[AES256_GCM,data:TMeguXfanISeyvsay9SBqm3SSGKpp5nCkqhHblf0QHNzHWGQKwpORmWfOtVfgOh9qdDqq8wYBpXznmbvixjV,iv:IR/rMoAIvZCw9FURmau4+g8c3pvI9BRs7v1NJ5ia4jI=,tag:kjwf6RN5HN8I2sUhDcr4UQ==,type:str] restic: - repo: ENC[AES256_GCM,data:/vybkTU7LMWSlco9W2pJouU9wm4okXClSHXQMCA6SGIHWp4Ppl6C+jS4sNJALc6ntKzcEHyWO/R3JPjQKjZNH4YtrnNQp/ZY9g==,iv:gAvp6blg5JuBKzLw6YSgM1Uc24Aesov3ttCRXZXBvJw=,tag:pvH1y6BFOl7jIn/qQejUbQ==,type:str] - password: ENC[AES256_GCM,data:5eIIBtGtBFwcAQ+ZwTYOtg==,iv:3GEM8Imu0i1aTwwSspvz2EzwJOXUC/b15hzkFFuZ+YY=,tag:wscba+nMtshldgUtcEKnOw==,type:str] + repo_url: ENC[AES256_GCM,data:GMHbrjgwajnYSiqtoYaKiFT/aDWDwlzEkvMLPzYf7C9PvLr7T4zeWyAA9//8huldyxO3+nk6O9lR9ORZKZfb8/MYB7nRB03sZQ==,iv:6uBhsksOGDjoc13U2xWLz7I+0fzGRhnw0nStACqlnug=,tag:uhH28NYq+ly1bmCV/cpxkQ==,type:str] + repo_password: ENC[AES256_GCM,data:jRHNgOk5ChWdqMKsd/V4Xg==,iv:wrgF5pau/RylG1nmJYmvrZ02o67qkkT5PrZAQlXb6Qo=,tag:X0WVpMqi8xeoATss/sSPMA==,type:str] sops: kms: [] gcp_kms: [] @@ -33,8 +33,8 @@ sops: dnFBa0lDWWZtS1BHdzBoVzNTaGNkSEEKi/W1n7RT8NpTp00SBMwxsUJAPDhumJ/i V2VnaSNwouD3SswTcoBzqQpBP9XrqzjIYGke90ZODFQbMY9WDQ+O0g== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-09-30T07:19:35Z" - mac: ENC[AES256_GCM,data:WSGvA1RkChrD07Sf4BFVMbdTXQYxAHeGGQ52e+pnPh0lZPOzMc9sLDrBPqDK2OfrHC+hK8RC7FxQTGs6G/oBB4nUzIZPn9WycTiU5elwWDfktizH0gr3EJDm7Gs+bTWQpwdoJZGZ8XErK+yegCaKL5cSOSTlBBbQOnZfnoNBg5c=,iv:xyJRFfxHC2xV0ro4CbdOPau1zORxA64OqpvKr4aFZvQ=,tag:c9NA90d5WTK2pfxwoyOX5A==,type:str] + lastmodified: "2024-11-28T03:57:35Z" + mac: ENC[AES256_GCM,data:xjZrlwfWLtZNYfH+KiE2ICt9Jo4nx/LKaEYi/ECN/Od+ZTjety0V6RJ/RfmI6q3K1WMj0sAGc56hCZ0iOn25L8wK6dc14hZVoSwwbIiQ7hTQE5LcK+NbXNmy3r/YC855DHG9kE08eYGHdNcBbckZg3HhkHQ9UYS/Ox/QFFuBa5Q=,iv:N3AW+sr9ET3c/ArXr176haRewYFsfgsNn+hkC0MDJwA=,tag:SCikn+F8btuSBswV+oCdXg==,type:str] pgp: [] unencrypted_suffix: _unencrypted - version: 3.9.0 + version: 3.9.1 diff --git a/machines/massicot/services/restic.nix b/machines/massicot/services/restic.nix index c8c28be..c205989 100644 --- a/machines/massicot/services/restic.nix +++ b/machines/massicot/services/restic.nix @@ -12,22 +12,26 @@ let in { sops.secrets = { - "restic/repo" = { + "restic/repo_url" = { sopsFile = ../secrets.yaml; }; - "restic/password" = { + "restic/repo_password" = { sopsFile = ../secrets.yaml; }; }; custom.restic = { enable = true; - repositoryFile = config.sops.secrets."restic/repo".path; - passwordFile = config.sops.secrets."restic/password".path; paths = [ "/backup" "/mnt/storage" ]; + backupPrepareCommand = [ + (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite") + (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3") + (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite") + (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db") + ]; }; services.postgresqlBackup = { @@ -38,12 +42,6 @@ in }; services.restic.backups.${config.networking.hostName} = { - backupPrepareCommand = builtins.concatStringsSep "\n" [ - (sqliteBackup "/var/lib/hedgedoc/db.sqlite" "/backup/hedgedoc" "db.sqlite") - (sqliteBackup "/var/lib/bitwarden_rs/db.sqlite3" "/backup/bitwarden_rs" "db.sqlite3") - (sqliteBackup "/var/lib/gotosocial/database.sqlite" "/backup/gotosocial" "database.sqlite") - (sqliteBackup "/var/lib/kanidm/kanidm.db" "/backup/kanidm" "kanidm.db") - ]; extraBackupArgs = [ "--limit-upload=1024" ]; diff --git a/machines/minimal/default.nix b/machines/minimal/default.nix new file mode 100644 index 0000000..7bc5549 --- /dev/null +++ b/machines/minimal/default.nix @@ -0,0 +1,25 @@ +{ + lib, + ... +}: + +{ + imports = [ + ./hardware-configuration.nix + ]; + + boot.initrd.availableKernelModules = + [ + ]; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.ens3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; +} diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index bcfdca7..b83e212 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -4,6 +4,7 @@ ./common-settings/autoupgrade.nix ./common-settings/nix-conf.nix ./common-settings/proxy-server.nix + ./disk-partitions ./restic.nix ./vaultwarden.nix ./prometheus diff --git a/modules/nixos/restic.nix b/modules/nixos/restic.nix index 0926fad..bef9c44 100644 --- a/modules/nixos/restic.nix +++ b/modules/nixos/restic.nix @@ -6,63 +6,150 @@ ... }: let + inherit (lib) + mkEnableOption + mkOption + mkDefault + mkIf + types + getExe + ; cfg = config.custom.restic; + mapBtrfsRoots = + rootDir: + let + backupDir = lib.removeSuffix "/" "/backup${rootDir}"; + slash = if rootDir == "/" then "" else "/"; + awk = getExe pkgs.gawk; + continueIfInExclude = '' + exclude_subv="${toString cfg.btrfsExcludeSubvolume}" + found=false + for subv in $exclude_subv; do + if [[ "$subvol" == "$subv" ]]; then + found=true + echo "$subvol is in exclude subvolumes, skipped" + break + fi + done + $found && continue + ''; + in + { + backupPrepareCommand = '' + echo "Creating snapshot for ${rootDir}" + subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -o "${rootDir}" | ${awk} '{print $NF}') + mkdir -p "${backupDir}" + ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}" "${backupDir}/rootfs" + for subvol in $subvolumes; do + ${continueIfInExclude} + [[ /"$subvol" == "${backupDir}"* ]] && continue + + snapshot_path=$(dirname "${backupDir}/$subvol") + mkdir -p "$snapshot_path" + + echo "Creating snapshot for subvolume: $subvol at $snapshot_path" + ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r "${rootDir}${slash}$subvol" "$snapshot_path" + done + ''; + + # Note that all the manually created snapshots under backupDir will also be cleaned + backupCleanupCommand = '' + # Only find snapshots under backup directory + subvolumes=$(${pkgs.btrfs-progs}/bin/btrfs subvolume list -s -o "${backupDir}" | ${awk} '{print $NF}') + for subvol in $subvolumes; do + echo "Removing snapshot for subvolume: $subvol" + ${pkgs.btrfs-progs}/bin/btrfs subvolume delete "$subvol" + done + ''; + }; + + btrfsFs = lib.attrsets.filterAttrs ( + n: v: v.fsType == "btrfs" && ((isNull cfg.btrfsRoots) || (builtins.elem n cfg.btrfsRoots)) + ) config.fileSystems; + btrfsFsRoot = builtins.attrNames btrfsFs; + btrfsCommands = (map mapBtrfsRoots btrfsFsRoot); in { options = { custom.restic = { - enable = lib.mkEnableOption "restic"; - paths = lib.mkOption { - type = lib.types.listOf lib.types.str; + enable = mkEnableOption "restic"; + paths = mkOption { + type = types.listOf types.str; default = [ "/home" "/var/lib" ]; }; - prune = lib.mkEnableOption "auto prune remote restic repo"; - repositoryFile = lib.mkOption { - type = lib.types.str; - default = ""; + prune = mkEnableOption "auto prune remote restic repo"; + btrfsRoots = mkOption { + type = types.nullOr (types.listOf types.str); + default = [ "/" ]; + description = '' + Includeded btrfs roots. `null` means snapshot all btrfs filesystems under config.fileSystems. + ''; }; - passwordFile = lib.mkOption { - type = lib.types.str; - default = ""; + btrfsExcludeSubvolume = mkOption { + type = types.listOf types.str; + default = [ + "nix" + "rootfs" + "swap" + "var/tmp" + ]; + example = lib.literalExpression '' + [ "var/tmp" "srv" ] + ''; + }; + backupPrepareCommand = mkOption { + type = types.listOf types.str; + }; + backupCleanupCommand = mkOption { + type = types.listOf types.str; }; }; }; - config = lib.mkIf cfg.enable { - services.restic.backups.${config.networking.hostName} = lib.mkMerge [ - { - repositoryFile = cfg.repositoryFile; - passwordFile = cfg.passwordFile; - exclude = [ - "/home/*/.cache" - "/home/*/.cargo" - "/home/*/.local/share/Steam" - "/home/*/.local/share/flatpak" - ]; - timerConfig = { - OnCalendar = "00:05"; - RandomizedDelaySec = "5h"; - }; - pruneOpts = lib.mkIf cfg.prune [ - "--keep-daily 7" - "--keep-weekly 5" - "--keep-monthly 12" - "--keep-yearly 75" - ]; - paths = lib.mkDefault cfg.paths; - initialize = true; - } - (lib.mkIf (config.fileSystems."/".fsType == "btrfs") { - backupPrepareCommand = '' - ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r / backup - ''; - backupCleanupCommand = '' - ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup - ''; - paths = map (p: "/backup" + p) cfg.paths; - }) - ]; + config = mkIf cfg.enable { + services.restic.backups.${config.networking.hostName} = { + repositoryFile = config.sops.secrets."restic/repo_url".path; + passwordFile = config.sops.secrets."restic/repo_password".path; + exclude = [ + "**/.cache" + "**/.local/share/Steam" + "**/.local/share/flatpak" + + "**/.cargo" + "**/.rustup" + + "**/node_modules" + + "*.pyc" + "*.pyo" + "**/__pycache__" + "**/.virtualenvs" + "**/.venv" + + # temp files / lock files + "*.sqlite-wal" + "*.sqlite-shm" + "*.db-wal" + "*.db-shm" + ]; + timerConfig = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + pruneOpts = mkIf cfg.prune [ + "--keep-daily 7" + "--keep-weekly 5" + "--keep-monthly 12" + "--keep-yearly 75" + ]; + paths = mkDefault cfg.paths; + initialize = true; + backupPrepareCommand = lib.strings.concatLines cfg.backupPrepareCommand; + backupCleanupCommand = lib.strings.concatLines cfg.backupCleanupCommand; + }; + custom.restic.backupPrepareCommand = map (x: x.backupPrepareCommand) btrfsCommands; + custom.restic.backupCleanupCommand = map (x: x.backupCleanupCommand) btrfsCommands; }; }