machines/thorite: add more scrapes, alerts; add restic backup;

machines/thorite/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./hardware-configurations.nix
     ./monitoring.nix
+    ./restic.nix
   ];
 
   config = {

machines/thorite/monitoring.nix

@@ -14,7 +14,19 @@ with my-lib;
 
     custom.monitoring = {
       grafana.enable = true;
-      loki.enable = true;
+      loki = {
+        enable = true;
+        rules = {
+          sshd_closed = {
+            condition = ''count_over_time({unit="sshd.service"} |~ "Connection closed by authenticating user" [15m]) > 25'';
+            description = "More then 25 users have tried logging in the last 15 min without success";
+          };
+          unusual_log_volume = {
+            condition = ''sum by (unit) (rate({unit=~".+"}[5m])) > 80'';
+            description = "Unit {{ $labels.unit }} is logging at an unusually high rate";
+          };
+        };
+      };
       promtail.enable = true;
     };
 
@@ -30,7 +42,10 @@ with my-lib;
         blackbox.enable = true;
         node.enable = true;
       };
-      ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]);
+      ruleModules =
+        (mkCaddyRules [ { host = "thorite"; } ])
+        ++ (mkNodeRules [ { host = "thorite"; } ])
+        ++ (mkBlackboxRules [ { host = "thorite"; } ]);
     };
 
     services.prometheus.scrapeConfigs =
@@ -39,8 +54,6 @@ with my-lib;
           "la-00.video.namely.icu:8080"
           "fre-00.video.namely.icu:8080"
           "hk-00.video.namely.icu:8080"
-          "49.13.13.122:443"
-          "45.142.178.32:22"
           "home.xinyang.life:8000"
         ];
         passwordFile = config.sops.secrets."prometheus/metrics_password".path;
@@ -52,6 +65,11 @@ with my-lib;
           address = "weilite.coho-tet.ts.net";
           port = 8082;
         }
+        {
+          name = "restic_rest_server";
+          address = "backup.xinyang.life";
+          port = 8443;
+        }
         {
           inherit passwordFile;
           name = "gotosocial";
@@ -72,6 +90,7 @@ with my-lib;
         }
         {
           name = "loki";
+          scheme = "http";
           address = "thorite.coho-tet.ts.net";
           port = 3100;
         }
@@ -90,11 +109,11 @@ with my-lib;
       ++ (mkBlackboxScrapes [
         {
           hostAddress = "thorite.coho-tet.ts.net";
-          targetAddresses = probeList;
+          targetAddresses = probeList ++ [ "49.13.13.122:22" ];
         }
         {
           hostAddress = "massicot.coho-tet.ts.net";
-          targetAddresses = probeList;
+          targetAddresses = probeList ++ [ "45.142.178.32:22" ];
         }
         {
           hostAddress = "weilite.coho-tet.ts.net";

machines/thorite/restic.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  sqliteBackup = fromPath: toPath: file: ''
+    mkdir -p ${toPath}
+    ${lib.getExe pkgs.sqlite} ${fromPath} ".backup '${toPath}/${file}'"
+  '';
+in
+{
+  sops.secrets = {
+    "restic/repo_url" = { };
+    "restic/repo_password" = { };
+  };
+
+  custom.restic = {
+    enable = true;
+    paths = [
+      "/backup/db"
+      "/backup/var/lib"
+    ];
+    backupPrepareCommand = [
+      ''
+        mkdir -p /backup/var
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r /var/lib /backup/var/lib
+      ''
+    ];
+    backupCleanupCommand = [
+      ''
+        ${pkgs.btrfs-progs}/bin/btrfs subvolume delete /backup/var/lib
+      ''
+    ];
+    btrfsRoots = [ ];
+  };
+
+  services.postgresqlBackup = {
+    enable = true;
+    compression = "zstd";
+    compressionLevel = 9;
+    location = "/backup/db/postgresql";
+  };
+
+  services.restic.backups.${config.networking.hostName} = {
+    extraBackupArgs = [
+      "--limit-upload=1024"
+    ];
+  };
+}

machines/thorite/secrets.yaml

@@ -1,5 +1,8 @@
 grafana:
     oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str]
+restic:
+    repo_url: ENC[AES256_GCM,data:tc7wYRN20sHxATTZYEBpf6tNafzq9vcvqdUHYJDmJIArxprNd6WiyqPXowzbksZcEi5JwSwwJH/MYminnPGtrR8erWZg8OB3,iv:/z7mF58tMAviscFWHd4NJw7UZlq7Bzz+LU88J+kE9qg=,tag:i97FP4SmmNXOuxylkHhYCA==,type:str]
+    repo_password: ENC[AES256_GCM,data:o3MbXJRwR5UE9uCELN2ejQ==,iv:cYPNjJAV7H2BNCuFLDJoJvPk+CFvagXJwW9LRAGc0G0=,tag:qF6Di2W+8kESCRAphC/c0g==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -24,8 +27,8 @@ sops:
             M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx
             mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-28T17:02:03Z"
+    lastmodified: "2024-12-03T08:18:54Z"
-    mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str]
+    mac: ENC[AES256_GCM,data:jqSt34avoMfL9g3LmvjrPTzW4xGLgX70CXI8qk4isaLbZ8FkxjVU8QY1ot9GZnFEQWUkReSuGD4gFxi8TjetlNdx0zDPcv6zGJUSfcYpyKDCqGdyL/2x8xnYtI2pWINBZxR/2XxT3cus39FJdXVcz3l7KX4DvYvm8t/D9+r4ef0=,iv:KY/OTbDOOD/bBDTIuIk1ck7wDxLogo2EKeSOfOe4j5o=,tag:B17iF5O32KDZfctubpXCng==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.1