diff --git a/machines/thorite/default.nix b/machines/thorite/default.nix index b85bab8..afe2e58 100644 --- a/machines/thorite/default.nix +++ b/machines/thorite/default.nix @@ -2,6 +2,7 @@ imports = [ ./hardware-configurations.nix ./monitoring.nix + ./restic.nix ]; config = { diff --git a/machines/thorite/monitoring.nix b/machines/thorite/monitoring.nix index ac6586f..164776e 100644 --- a/machines/thorite/monitoring.nix +++ b/machines/thorite/monitoring.nix @@ -14,7 +14,19 @@ with my-lib; custom.monitoring = { grafana.enable = true; - loki.enable = true; + loki = { + enable = true; + rules = { + sshd_closed = { + condition = ''count_over_time({unit="sshd.service"} |~ "Connection closed by authenticating user" [15m]) > 25''; + description = "More then 25 users have tried logging in the last 15 min without success"; + }; + unusual_log_volume = { + condition = ''sum by (unit) (rate({unit=~".+"}[5m])) > 80''; + description = "Unit {{ $labels.unit }} is logging at an unusually high rate"; + }; + }; + }; promtail.enable = true; }; @@ -30,7 +42,10 @@ with my-lib; blackbox.enable = true; node.enable = true; }; - ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]); + ruleModules = + (mkCaddyRules [ { host = "thorite"; } ]) + ++ (mkNodeRules [ { host = "thorite"; } ]) + ++ (mkBlackboxRules [ { host = "thorite"; } ]); }; services.prometheus.scrapeConfigs = @@ -39,8 +54,6 @@ with my-lib; "la-00.video.namely.icu:8080" "fre-00.video.namely.icu:8080" "hk-00.video.namely.icu:8080" - "49.13.13.122:443" - "45.142.178.32:22" "home.xinyang.life:8000" ]; passwordFile = config.sops.secrets."prometheus/metrics_password".path; @@ -52,6 +65,11 @@ with my-lib; address = "weilite.coho-tet.ts.net"; port = 8082; } + { + name = "restic_rest_server"; + address = "backup.xinyang.life"; + port = 8443; + } { inherit passwordFile; name = "gotosocial"; @@ -72,6 +90,7 @@ with my-lib; } { name = "loki"; + scheme = "http"; address = "thorite.coho-tet.ts.net"; port = 3100; } @@ -90,11 +109,11 @@ with my-lib; ++ (mkBlackboxScrapes [ { hostAddress = "thorite.coho-tet.ts.net"; - targetAddresses = probeList; + targetAddresses = probeList ++ [ "49.13.13.122:22" ]; } { hostAddress = "massicot.coho-tet.ts.net"; - targetAddresses = probeList; + targetAddresses = probeList ++ [ "45.142.178.32:22" ]; } { hostAddress = "weilite.coho-tet.ts.net"; diff --git a/machines/thorite/restic.nix b/machines/thorite/restic.nix new file mode 100644 index 0000000..ef21c66 --- /dev/null +++ b/machines/thorite/restic.nix @@ -0,0 +1,51 @@ +{ + config, + lib, + pkgs, + ... +}: +let + sqliteBackup = fromPath: toPath: file: '' + mkdir -p ${toPath} + ${lib.getExe pkgs.sqlite} ${fromPath} ".backup '${toPath}/${file}'" + ''; +in +{ + sops.secrets = { + "restic/repo_url" = { }; + "restic/repo_password" = { }; + }; + + custom.restic = { + enable = true; + paths = [ + "/backup/db" + "/backup/var/lib" + ]; + backupPrepareCommand = [ + '' + mkdir -p /backup/var + ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r /var/lib /backup/var/lib + '' + ]; + backupCleanupCommand = [ + '' + ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup/var/lib + '' + ]; + btrfsRoots = [ ]; + }; + + services.postgresqlBackup = { + enable = true; + compression = "zstd"; + compressionLevel = 9; + location = "/backup/db/postgresql"; + }; + + services.restic.backups.${config.networking.hostName} = { + extraBackupArgs = [ + "--limit-upload=1024" + ]; + }; +} diff --git a/machines/thorite/secrets.yaml b/machines/thorite/secrets.yaml index 60d475f..c246e2b 100644 --- a/machines/thorite/secrets.yaml +++ b/machines/thorite/secrets.yaml @@ -1,5 +1,8 @@ grafana: oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str] +restic: + repo_url: ENC[AES256_GCM,data:tc7wYRN20sHxATTZYEBpf6tNafzq9vcvqdUHYJDmJIArxprNd6WiyqPXowzbksZcEi5JwSwwJH/MYminnPGtrR8erWZg8OB3,iv:/z7mF58tMAviscFWHd4NJw7UZlq7Bzz+LU88J+kE9qg=,tag:i97FP4SmmNXOuxylkHhYCA==,type:str] + repo_password: ENC[AES256_GCM,data:o3MbXJRwR5UE9uCELN2ejQ==,iv:cYPNjJAV7H2BNCuFLDJoJvPk+CFvagXJwW9LRAGc0G0=,tag:qF6Di2W+8kESCRAphC/c0g==,type:str] sops: kms: [] gcp_kms: [] @@ -24,8 +27,8 @@ sops: M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-11-28T17:02:03Z" - mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str] + lastmodified: "2024-12-03T08:18:54Z" + mac: ENC[AES256_GCM,data:jqSt34avoMfL9g3LmvjrPTzW4xGLgX70CXI8qk4isaLbZ8FkxjVU8QY1ot9GZnFEQWUkReSuGD4gFxi8TjetlNdx0zDPcv6zGJUSfcYpyKDCqGdyL/2x8xnYtI2pWINBZxR/2XxT3cus39FJdXVcz3l7KX4DvYvm8t/D9+r4ef0=,iv:KY/OTbDOOD/bBDTIuIk1ck7wDxLogo2EKeSOfOe4j5o=,tag:B17iF5O32KDZfctubpXCng==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1