modules/monitoring: add loki and promtail

modules/prometheus: fix basic auth scrape
modules/prometheus: add basic auth
2024-12-02 14:44:26 +08:00 · 2024-12-02 10:43:43 +08:00 · 2024-11-30 17:35:46 +08:00 · 2024-11-30 12:56:54 +08:00 · 2024-11-30 12:54:51 +08:00
30 changed files with 732 additions and 386 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -10,6 +10,17 @@ keys:
  - &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
  - &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
 creation_rules:
+  - path_regex: machines/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-calcite
+      - *host-massicot
+      - *host-thorite
+      - *host-biotite
+      - *host-hk-00
+      - *host-fra-00
+      - *host-la-00
  - path_regex: machines/calcite/secrets.yaml
    key_groups:
    - age:
@ -25,6 +36,11 @@ creation_rules:
    - age:
      - *xin
      - *host-massicot
+  - path_regex: machines/biotite/secrets.yaml
+    key_groups:
+    - age:
+      - *xin
+      - *host-biotite
  - path_regex: machines/thorite/secrets.yaml
    key_groups:
    - age:
@ -32,11 +48,11 @@ creation_rules:
      - *host-thorite
  - path_regex: machines/dolomite/secrets/secrets.yaml
    key_groups:
-    - age:
+    - age: 
      - *xin
-      - *host-la-00
      - *host-hk-00
      - *host-fra-00
+      - *host-la-00
  - path_regex: machines/dolomite/secrets/la-00.yaml
    key_groups:
    - age:
@ -47,7 +63,6 @@ creation_rules:
    - age:
      - *xin
      - *host-hk-00
-
  - path_regex: machines/dolomite/secrets/fra-00.yaml
    key_groups:
    - age:
@ -58,15 +73,6 @@ creation_rules:
    - age:
      - *xin
      - *host-weilite
-  - path_regex: machines/secrets.yaml
-    key_groups:
-    - age:
-      - *xin
-      - *host-calcite
-      - *host-raspite
-      - *host-la-00
-      - *host-hk-00
-      - *host-massicot
  - path_regex: home/xin/secrets.yaml
    key_groups:
    - age:
--- a/machines/biotite/default.nix
+++ b/machines/biotite/default.nix
@ -1,12 +1,13 @@
 {
-  config,
  lib,
-  pkgs,
  ...
 }:

 {
-  imports = [ ./hardware-configurations.nix ];
+  imports = [
+    ./hardware-configurations.nix
+    ./services/gotosocial.nix
+  ];

  networking.hostName = "biotite";
  networking.useNetworkd = true;
@ -20,11 +21,28 @@
    address = [ "2a03:4000:4a:148::1/64" ];
  };

+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
  commonSettings = {
    auth.enable = true;
    autoupgrade.enable = true;
  };

+  custom.monitoring = {
+    promtail.enable = true;
+  };
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+  };
+
+  services.caddy.enable = true;
+  services.tailscale.enable = true;
+
  users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";

  system.stateVersion = "24.11";
--- a/machines/biotite/secrets.yaml
+++ b/machines/biotite/secrets.yaml
@ -0,0 +1,31 @@
+gotosocial:
+    oidc_client_secret: ENC[AES256_GCM,data:KVQxzs67sohax2h0Y/jjhnbY4fetrdVvWhBGbqgDSGgBC7QazrOmTA++BSRzMmVv,iv:HIRMc56aLanqQRTWH9E0wzzXymImi0pxK/ccPEP8Fcc=,tag:PMhOLeE3mKIIQveRdfpgpA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxVXpUNXA3eEZEeGxpMmZT
+            L0lPUzYzNXlrS2JDbWlYNzJiYmwwYm1PSjFNCjAzSGluME1hd1Fnc0ZCNUhUMzdU
+            UHkwbmxwdTdVOFhIYUo3N0laVlJRV0EKLS0tIHR5NDJqQnI3ZkFGcmwwaHZwOGd2
+            Y2gvVTRMc2RSd1UxWUdEWVZDRm5VbHMKLYJ59s2MDDokJRAAXoTAL1VTU4WKY8qS
+            GiXZu954JzacAR9Ey2GQTFdMN73Aw+PbiWw6cph33gZaOQt9/QA92w==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3djErT0VVOU9ydmpjL01a
+            aDFQa2JiMVBURzhCZ0NBUDdaMDZCV2piUjI0ClBmSGJIallnTzdmV3RYZlNBK0Ji
+            K21qRkg0SDY3WkZ5bXFrWitBSGNEQ1EKLS0tIGhHMGRsZGNaL2hNWFdKUTJUUk1G
+            RzBMVDNjS29SUkdRK3dIV01sU0hYR3cK1SbvKAM6Gpsffv3HIi/WtWnCZUBic0AT
+            ZRv4pvJBx1oxWsKIHW0t6VrqWMQ+suup8p6dW+h5HE8Z4ciIMrXLEg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-12-02T05:10:32Z"
+    mac: ENC[AES256_GCM,data:ZAdFsjVuk1Fiv+DKmHrc1yu1XQpRDmRHaQhu5hduSZUa1W1cXdTlChvIW5vADFg5tVCjuYptuLvCMW+ZSQeqqG2ntHHZ+IkuovZzKFuc+BIiL/jF2ZzbyJ7X4Wj1GziCScHVxx98dgbpFoufHe6N3wCaHmngo1RYsY5N1RRbRdU=,iv:5IMQ0kOX9UAOm8bcsQRyu6zu8GJjvnHFufCNjY0s9UI=,tag:zBEPSR9DZDpwbCaIka8mXA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.1
--- a/machines/biotite/services/gotosocial.nix
+++ b/machines/biotite/services/gotosocial.nix
@ -0,0 +1,46 @@
+{ config, ... }:
+{
+  sops.secrets."gotosocial/oidc_client_secret" = {
+    owner = "gotosocial";
+  };
+
+  sops.templates."gotosocial.env" = {
+    owner = "gotosocial";
+    content = ''
+      GTS_OIDC_CLIENT_SECRET=${config.sops.placeholder."gotosocial/oidc_client_secret"}
+    '';
+  };
+
+  services.gotosocial = {
+    enable = true;
+    settings = {
+      log-level = "info";
+      bind-address = "127.0.0.1";
+      port = 19571;
+      host = "gts.xiny.li";
+      account-domain = "xiny.li";
+      letsencrypt-enabled = false;
+      instance-expose-public-timeline = true;
+      oidc-enabled = true;
+      oidc-idp-name = "Kanidm";
+      oidc-issuer = "https://auth.xinyang.life/oauth2/openid/gotosocial";
+      oidc-client-id = "gotosocial";
+      oidc-link-existing = true;
+    };
+    environmentFile = config.sops.templates."gotosocial.env".path;
+  };
+
+  services.caddy = {
+    virtualHosts."https://gts.xiny.li".extraConfig = ''
+      encode zstd gzip
+      reverse_proxy * http://${config.services.gotosocial.settings.bind-address}:${toString config.services.gotosocial.settings.port} {
+          flush_interval -1
+      }
+    '';
+    virtualHosts."https://xiny.li".extraConfig = ''
+      redir /.well-known/host-meta* https://gts.xiny.li{uri} permanent  # host
+      redir /.well-known/webfinger* https://gts.xiny.li{uri} permanent  # host
+      redir /.well-known/nodeinfo* https://gts.xiny.li{uri} permanent   # host
+    '';
+  };
+}
--- a/machines/calcite/configuration.nix
+++ b/machines/calcite/configuration.nix
@ -18,7 +18,6 @@ in
  commonSettings = {
    auth.enable = true;
    nix = {
-      enableMirrors = true;
      signing.enable = true;
    };
  };
--- a/machines/dolomite/common.nix
+++ b/machines/dolomite/common.nix
@ -2,6 +2,7 @@
 {
  config = {
    sops = {
+      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
      secrets = {
        wg_private_key = {
          owner = "root";
@ -33,6 +34,10 @@
      node.enable = true;
    };

+    custom.monitoring = {
+      promtail.enable = true;
+    };
+
    services.tailscale.enable = true;

    commonSettings = {
--- a/machines/dolomite/fra.nix
+++ b/machines/dolomite/fra.nix
@ -62,7 +62,5 @@
    address = [ "185.217.108.59/24" ];
  };

-  custom.prometheus.enable = false;
-
  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
 }
--- a/machines/dolomite/secrets/secrets.yaml
+++ b/machines/dolomite/secrets/secrets.yaml
@ -1,6 +1,6 @@
 sing-box:
-    password: ENC[AES256_GCM,data:aifvj/rBvmIF6M4SJ6j4rkw0J0oBGUmO,iv:C9KlVngh74z/VjjOGxnlpA4CqFv7TCSD3KSm2l/xGB4=,tag:10zUgbP2exTQ4KK0zeMM2A==,type:str]
-    uuid: ENC[AES256_GCM,data:ZPEqllAXeLMyVEp/6+9LSL346J2tiuM5tYs404/vp9rnkrvc,iv:Oy/U1c2sW5a2eQQxXAEjqaE85xX5rFapz9k/DtcZR+w=,tag:BHU+ScDBeWnctkDBRnm+4g==,type:str]
+    password: ENC[AES256_GCM,data:qCc1v8nAL0oYisRinMDXGrBQA+r6XNoa,iv:eTxtad4kEdE28XqnrZEek8BtXNY1rNgLvGLxlMzRtl4=,tag:s/shWAkYE4DSnScpTY8ulQ==,type:str]
+    uuid: ENC[AES256_GCM,data:lEpz15sLOVrGDzQwTJyS+tFJY0bMeO265bxocWAjB6qrvxYx,iv:lhk5jl/udUH3AZEuk5ffuvin/qhRUaOZ/3nk1Jaw+DI=,tag:4mKFIVKT+D47njfDsxe9iA==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -10,50 +10,41 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmdCtZK2FVRTh3YVd3dm9m
-            ZWR5VVIvS3VOSGh2cmg2ZUFrYmNIdVNLSTNVCjlhVlJER1BZMlRUd1RkYnpvTE9F
-            bExGa1NBWWR0enBmUFJYVVA4UlI1cUkKLS0tIC8wa3FGRnFldVdTdkpBb2xQc3BD
-            cTlhNHplRUoyS3pxNnF0TVlFTy9kdzQK4kDSzSV4ZnELvCsajGwvsc/vzua2hbI1
-            Vht7rmZ8Dl4Y3xEIXG7XVnWK2GOblpqZ/eza1T6kWEkXp2uCdQnM6Q==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1dVV0U3kwSmdnTU1HcGpr
-            U2FKZVV1c1R6a3ovRGxoOUlrcUNWUUFHN25ZClBBTUZGeTc0Tkx1OXdaK1p6aWpr
-            aSsvN0ZDR1V3VnVrb1FBYzdHSTNXOVkKLS0tIFlSUk5LT1hVUUd1aVg1eVNTUURX
-            OXRVVmNRWEhmVXZkWC9HNTUyUTNrMlUK370K3D1vU97vHV9aGjYrFOIJzmOQAnzH
-            QR6XsOkM0FRvSkhTsEZ3qC4Wd2MTIyRzHYPKvZmz9LufIr1N/JFj1Q==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQT0YyeXI4d2o4V0lWUE4x
-            ZXZWWDFiakdqNlU5RWt6QUdxYVRSZzQyZkZBCi9Tdm5wRXB2cTYxdnVYRXJaS0d0
-            Lzg3VWpqQ1NOb1NTYXE4RGVRZVZoM1UKLS0tIFdGM01VU3FEc0ZyeEN3bVM1WEZq
-            M3BFa1hoWkQyRkJqSlZiTnBwQWphemcKLTAza2y96h+IyWB2EN6e4WIFQqeL5E7p
-            CDmHr+hSt6u9cr8C/etljxGMbKf9GqFOeuCyPugrJGdu4/qlR5iE0g==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxV0RYNXFMdlhqc0RhdHEx
+            bkNXT0tYYmpTK2NyUnpKRjQzVkpTMk4yVHlFClBaVHZoVXlqRXFxYStzR2U0MzVG
+            OHI0Qjl0amw0V2tneWtrUHpSYVg3VmMKLS0tIEpneDFuVWZ2TFUwN0QxZWJnVEE3
+            SEhGMG9ac3gyb21Sa3V0cnB5SnppM1EKzfuKBAjPChde2UAEib3yE5Dczv3/UePL
+            rHHxxSr6kIPIwtcjJpJJxqndLSCegXaomZukxuble3Xt4Nl4sVhaFg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFa1RHN2s4ajYzZmwvUlN2
-            c05SdERTTEhPRnJWOUF6TExIMnBEZkVMb1I4CkxBeTRQWmZEOGNrcFlGV2wrMkhI
-            QnAwSzZPaWNWbmdnZmFjZVJyRVdzN2cKLS0tIHVMU3Z6a1MrV3BVV1hqbEdYODJu
-            cGgvNU05eGx4alRNT2d5MWp6Q3lWZDAKQ+D1niMzaso/lQwdmepvACF8/SDEt2mQ
-            7nTRVJIpjGPTxO4ezcQWUGej+BSEnOoZno3epoIXLNlwDnHOAawTWQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1ci91a1lQa0NXUFhZNWEr
+            cHlsQ093NHpESGdZN2xvdEUyZS9SKzBvSkhvClBZNG82OGR1WTZUUXhCb0ZyYmVX
+            UlZlaHNxL3Z0ZjZ2dVRoOEJibWVZR0EKLS0tIFpQMlQyaTY0bHVsUk9nekh4R2dK
+            YkhMdG9MbUpDZzJvcXE0bkpLeXZVT3cKLCgizqmjO1hueLvvAWVyZ9dPQcYOQHwW
+            pE//uiFFpjRsXLVB556ZyGYHn4osTfq73XYqvpsE4gsxT2scGxP/ZQ==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNcHNReHZibVlrNUtncnl1
-            SzczRGVFdUNvcFdqeWpZUk5FL0hwOS9LT3l3CnFLdXozcUxXYUpjUXJZWEtjMXo3
-            d28reWd0Z1Y0NWdBTG1MTkRGSEphY2sKLS0tIGw5U3NiOU1DNitUd0x5SkJ3SHFj
-            RVpWNDNUb2d1SEZpQlFBK2tFVjFzU0kKtI7e+kkiBm1L/WzkBApRI8IIo3gHdrE1
-            fzR+sbYEHWf95iEmb/oGlH++TrFW/zRXEyWPAi4ORTs7s/Ql1UC4Wg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4VmdsdHJJc0pMOVFUSmV5
+            MUJaNTVWZUVrd2RrMUQxYVA2UlRlallwSmxJCjVOQWZESnViZVMxTTZPMElocm1C
+            TGlsOW90UytISDlGQi9zaGlZQ3BPamcKLS0tIDQwMW9WbUl2c29sVWxSWUk3bHAr
+            R0tTMHlPUlgxNVg0YlFyMm1kSm9ReHcKCMO2+wSj5OQJ+ClRsPADL9Zfg7oN6AzJ
+            IgKibbO2MGx/S+6x5K/QGEvaFWqh6bAWDgvdq/9I1kaO+fMpsmMqCA==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-22T05:51:19Z"
-    mac: ENC[AES256_GCM,data:LPUb7YbELPsgYX+LvfuGdiNG1B5ZrvyRVZL9UiMHoJMDHaWpDGCQkT1bk5jEOewwFh+StK560UsPK4uW0+SqADJO80rmM3xfqlp5Ul2HJ8mU19l5C6FLpv2REIzhCp333rNJJlyhn3H6GZgMaWnjjLSX9XMOVaq1iz6Qt0P76SI=,iv://EdDr5D51RFuucq8gkei2RC7H2bkRYxP/7lerp9dtk=,tag:JrRQ19sZ0OAZouMgOiU/tQ==,type:str]
+        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxR0tzQ2JuZkRrMGlWN0JJ
+            U3pwdHBmQ1N2NUlyT0s4REpmVFEyRk9XeFdrCkZQdXRPMktjYnZqc0trOGtNeHd1
+            QjZXZlozaVhYRUZ1TzQ0QVRxb20xZEUKLS0tIEF4WVh6VTFVVVVuajlXUGRSS2tS
+            K1F1SzI2NFNIKzlreVBXSjAxaUxQd28KFaf1uu7OlqIe0TirJFgS3iPjhXPyfNDE
+            m2XUjzdXp+chJCzVOFvpYStqz+e08ADEc+jp3YsTLcxyqvXhQdyL/Q==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-12-02T05:26:17Z"
+    mac: ENC[AES256_GCM,data:K94zFWPWGUisLCqDjSLs17QxHXPH4tPU/98Sb4lCnt7IRAIn14x/T+BnInY/DK+DOVLLtzSfuN0kgzzGjSzwJx5Vq1G3MkhngRQQRT9dvODTCMAw6lPt98Ofw1CEEsFQnpYo9zIUlCGKg2YPKFLqE7OjkPxqw7VYvgzr5dDw58s=,iv:3xcJfNX5v/e9HgZt3UrHs2/C5ivaBV1rXKIBs9hKKFg=,tag:RQPQQ1cmZiOpQjUwqnzZQA==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/massicot/kanidm-provision.nix
+++ b/machines/massicot/kanidm-provision.nix
@ -108,6 +108,20 @@
          ];
        };
      };
+      gotosocial = {
+        displayName = "GoToSocial";
+        originUrl = "https://gts.xiny.li/auth/callback";
+        originLanding = "https://gts.xiny.li/auth/callback";
+        allowInsecureClientDisablePkce = true;
+        scopeMaps = {
+          gts-users = [
+            "openid"
+            "email"
+            "profile"
+            "groups"
+          ];
+        };
+      };
      # It's used for all the clients. I'm too lazy to change the name.
      owncloud-android = {
        displayName = "ownCloud Apps";
@ -147,7 +161,7 @@
      immich = {
        displayName = "Immich";
        originUrl = [
-          "https://immich.xinyang.life:8000/api/oauth/mobile-redirect"
+          "https://immich.xinyang.life:8000/api/oauth/mobile-redirect/"
          "https://immich.xinyang.life:8000/auth/login"
          "https://immich.xinyang.life:8000/user-settings"
        ];
--- a/machines/massicot/services.nix
+++ b/machines/massicot/services.nix
@ -43,6 +43,10 @@ in
    environmentFile = config.sops.secrets.hedgedoc_env.path;
  };

+  custom.monitoring = {
+    promtail.enable = true;
+  };
+
  custom.prometheus.exporters = {
    enable = true;
    blackbox = {
--- a/machines/netdrives.nix
+++ b/machines/netdrives.nix
@ -1,22 +0,0 @@
-{ pkgs, config, ... }:
-{
-  sops.secrets = {
-    autofs-nas = {
-      owner = "davfs2";
-    };
-    autofs-nas-secret = {
-      path = "/etc/davfs2/secrets";
-    };
-  };
-  fileSystems."/media/nas" = {
-    device = "https://home.xinyang.life:5244/dav";
-    fsType = "davfs";
-    options = [
-      "uid=1000"
-      "gid=1000"
-      "rw"
-      "_netdev"
-    ];
-
-  };
-}
--- a/machines/osmium/default.nix
+++ b/machines/osmium/default.nix
@ -51,7 +51,7 @@
    };

    commonSettings = {
-      nix.enableMirrors = true;
+      nix.enable = true;
      auth.enable = true;
    };

--- a/machines/raspite/configuration.nix
+++ b/machines/raspite/configuration.nix
@ -9,7 +9,7 @@
  imports = [ ./hass.nix ];

  commonSettings = {
-    nix.enableMirrors = true;
+    nix.enable = true;
    auth.enable = true;
  };

--- a/machines/secrets.yaml
+++ b/machines/secrets.yaml
@ -1,14 +1,6 @@
-clash_subscription_link: ENC[AES256_GCM,data:uDaX2BE/qRdfXVtckX0VKpu0LN3j0YaxVIPbQt3tGAfdfqFqlp0IzFgNiZBIEcIltYkeEyqFSA0QnttoMb0QYe9f2rtgjztwk10SOGViGaeFWPfkdlHP04qhm5OOOddi3OwT5rUNwvBU79AdCnLJ9QwqMbOaNm/JTtbkcjf8huxc2UcYAQcY/YNJ7aTEhWIw98Ab85aih+w=,iv:pZ189IPPCBjscXzEdgQCRdFlls3TniwDfNCd+H1FFaQ=,tag:dMmGZvppWtkc82b5dTnJwg==,type:str]
-autofs-nas: ENC[AES256_GCM,data:LnCKGKARx6Vd99VwAX/6PXOJwo+a7GP8fNmM9yuuC2xITGxtWCsDdOZL1+IA5LS/gbOYINgQWDzWirJF3LCP27BQeLwXYpD7/UAwwVI=,iv:QJzsS5a6vWeoBxkB13yXdVbyn0tt2QTvqj0LaHn6S2g=,tag:D/JKXQIw1EzIh3wjGhHgHg==,type:str]
-autofs-nas-secret: ENC[AES256_GCM,data:gbOizRZAvh79HlJWIWeKTk79Ux311XGL1eIswc0P2U2huCibD/ji3kOlSjZXENG+fJQKNz2AlDTk3g2cQQ==,iv:UCaGeE8j4RqJzA0xhu3oB2xvzombzQD3fjLKCWd5fDg=,tag:II4eEMr7f2TDUl1qUcDYXA==,type:str]
-github_public_token: ENC[AES256_GCM,data:6Gt+oJcCRHeoLK7CRndMMbszTXSEbnN0nQzsVOnl/+zB4hxbEPD5k/vkkl+cZ/qmxdxFXV0OOsYvktn44Yv1DMUE3mkB0hcAdoyPwLuYM7W3RpOoW3OktH8DRCUi6msvFp3ykpdmIl9WyjVhc/lMwTaYJQyRh1ue,iv:PJSFtJBelyc3rzd6hqjMp+ciU2Q3FTOEXsiq5F2KKTY=,tag:Y/stRg6kwyjjIFZCXS/peg==,type:str]
-singbox_sg_server: ENC[AES256_GCM,data:SF2ja6W4TwThwoug5x2KTA==,iv:Vx9wNTdVHkReux4YeQY+0VkC1Wqg/CRkY7frVY/3e50=,tag:7XA9KSoR0GA6FoYRhCv4BQ==,type:str]
-singbox_jp_server: ENC[AES256_GCM,data:S3Bs5yVMzyz6vD51GYElOM5h,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:o9d55cZuWmX4NDYexWjvYQ==,type:str]
-sing-box:
-    password: ENC[AES256_GCM,data:xyqmoJEDI5959zHPTVelln/iThtoeDwS,iv:rLyqJsE/4JDf08RlMLLPh+MKJkba9bL0z8jx6bTEfgc=,tag:cgLHdeLIyPvLhRNaVcQ0TQ==,type:str]
-    uuid: ENC[AES256_GCM,data:lWBCM5wyz6BcUUHdvynkn5y166Kk15jO0EhWUDuhXXhrve5l,iv:RmDJYFnYqIEIShLn25sf4h8AO2E3+3Xa2U9Mff+Xk2w=,tag:SN0DUdwZXKO/VEnozrr5mA==,type:str]
-grafana_cloud_api: ENC[AES256_GCM,data:eEvPAwtThK1FMhbrnmSo89+GlWZAF+LQRMLXA2C6f1vR7ZPlXJZGWzjYwDcPlnpiC737/cG14M4kZqvPGBuNub5A83rBS/+FeebvGDIF59L5PC1Ys1jWBB9YRI/L9EU0tvwTTUCvLRA9j28n7Jw7wR6mWXm63XA+OMu8/UbTwbeV/WUQn8vnwqadSUdCnNKJXMsAY+q9t/st0DPm5+aNxA==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:87C+0FVvzDIowE0+QpY1zA==,type:str]
-private_dns_address: ENC[AES256_GCM,data:YJxNOH4hsZHResvANEqJRTANhnL4PLp/Pmi/PhgtSTbTKiJKPqudhTEkNg==,iv:8+qG5rQXAKfrykEjt9qrbtyNaBuKvi7EaIWouRqEipY=,tag:VH0w5ZbXcWFGZ9GLavm7/w==,type:str]
+prometheus:
+    metrics_username: ENC[AES256_GCM,data:/CQfOA==,iv:BjhB+uLfjmYHdgpc/+tDJXJ8C1EK9kngQWbo4NleOmE=,tag:JCdqyqGLRh09T25vmufiZw==,type:str]
+    metrics_password: ENC[AES256_GCM,data:q/xMPuNtlcUFewMdVu6w2Q==,iv:xLohdb5tdxevYFckZoacjSJp2rZ53QKLxK6u3mc3mDw=,tag:B4LrObH1DsnnD5CcuOPOyg==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -18,77 +10,77 @@ sops:
        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiRzZVNGFocUN2VzZLTmJz
-            WlJnUmxhZS92citDRkVZVnJZQU9YWVZORlNjCkgzeWl5dTl1YmpjZGt2anF3dGgr
-            K1hOSTRmakNrZ2JoNit3NDIzK1FCcWMKLS0tIEdqY3VvR3gxd1JoQlhPR3JvcXBF
-            K2g4VFpqUEF5RTQ3cmpUSG0xajN2bUUKMuwx5cO1nHokV1NOloXfl9wTBN/+/Rlq
-            UJKP/qaI23tpyMXN1U40iF20ecO1U5Ad8wAQ61C/tldSVULizDihpw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwMHB1bFQ3dWJIU3NiOVVP
+            Yi9LZE1PTVdMY1BqS1JHV3VPLzZIY0hGK0NZClNlclVXKzBvNTBrTlhiR0VsaVoz
+            RlVLNVBEVDgzSXB5ZGxDd3hqNDh2V2MKLS0tIEhBZHFUY3c2VXJBVEVKamZ6TzBa
+            MlFsNnVEV0xCdlJoRnBhUHF2MmswUEUKNYD9zssGBy9SaKeOMvTz71B6KMPW87cM
+            tFJzgnQceEQF658lVa5cCzG1gzraCgBtQU15XzC7e8zWI9CHquRRlQ==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYXlwdytVSm1SQzRubHdX
-            dHhrTWxyamo3OFRraEVRQ3plK1cwUWt0a1JRCkdqaVRTQ0NaTkdoMlpDT3Yyallq
-            eTd0bDViVTgwZGRTUmlYTzR0Y09iWWcKLS0tIEFlQnFPVFRVNlAxdExMekJ2b295
-            UUJkUUZCNUZnbkNFZHVBYXNHQklOL2MKujgh6REuAKu6ZLVA7atiWUqhnvYJnQjb
-            WsxCa9ZXZRgfbhcNlZ3qIKJpWWI/RMS17+Nm5yIl+2cSqe2UJMjZdg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNXQ4RVVRd3RYRkhUVExG
-            SHJON0hwSmJtUkYrd1dldHJRN2tPKzBsNlNjCi9xYVhaanF5TDU2Q0xadXNWR2tN
-            dHhQVkpRREFlRm5MM2pwVytEaGhHT0EKLS0tIE9sRUtLako3cnAxNm82RDhiWEVM
-            ZW1IMXkzYkhqbW1ZdVRabUlkK2oxSTgKHC22uQqMq+cJ7vrONkGgoH8snxGef6Ft
-            QbtoJziERjAhK6B7TOY8AJ3WVRpCzZN70HjLNYa+bMMNOvmlsVxfZg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSmZ5YUpFdzRNdWZVNmxJ
-            bm5ucUhVeTV2TkE4ZElkZ1N3aXc2eVEwMlRNCmxXRElPb2pGYzJFVnUxQkRtMlNF
-            cjgwUzh5UWNLTk01U0h2bHNpaXVzZkkKLS0tIDczUkkwTG8rL1V3UU9lenk4V2tl
-            TUxDd2huTllMRG9MZTJZdzRwaWxqUVEKLA3y+heUA8cK31LZzv5A1wtgf+sauuwE
-            7SGU3uYU650tJM3e6Lveo+JOAD7Z1jrAomT5Bub+jjSHnpeFC9yMbA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4enA2bnlrV3ViY1RHaTVS
-            Ym5VV005NFlXZUl2NDNXYXBoOHh0SGQ0YVNnCm1KdHBSeE9lQzZEM2hFZUwzRitS
-            K3BEWGhtWmxKc2RJd0FTMEs2b1ArOUkKLS0tIG5kaWc3U0o4SG9teXk5dVZWWjkz
-            cS9VMU5YbEl3UE5mODJ1THNLVEdVblEKNQF0b9r1XPD819Z6Uy0b9hT4Uek2tNWU
-            3z3H7V/UiB1TMW+qgs6BC6bDkDf7oG//qmZEdYF+lDXcNSwai25xyg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBreHJNRXlpOEh6YUxaSmJj
-            ZDlVdHh2b1p6aEs3eDAvbkk4WExxWmE5bDM0CkZzT2l1K09UbmNFNEpZUVY2NVlB
-            dVFYbnpvTjlUcTdZejMrelpscXRJQzAKLS0tIEVIaVByVmp0aUU1ZWJLajBhcjRk
-            QVZMRXBRVVhaY3JKZEJjMTdEeEVqcWMKT+DoevNQAxCrty2VkRDLWGFzs9GsW3F7
-            txz73tAceAIiocC1z7IV2TaYULYf7Z75HAje/SOTlGHBIDiVZ0vyLw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTTnZLTlZQRzc1enVEa1BN
+            SHdoSi9oOXk4UTV0SlRZS2tLS2FFL3VjNzNNClVWTTNKekF6T0RTUzdEeWhLbHoz
+            WFZKaHJEaVBWa04zRWRiVnJZRjU0YVEKLS0tIFJVL0FEemowS3V6MmsxbWJMU2I1
+            U2NnUnVKdFlRSGVzUFQ4ZFcwL0lWTlkKz1t3yqjgIdMWS/Nsy2nq3oCjOhGDP+UT
+            L+LAuFExJPV0qlsOG/kCGB/WtCJfnBvcp6vPDBLqjK8NllIX/iPI5g==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEZEJHQlJqMWxob1lxOUFK
-            dDZIN3FaNWR1L0gyN3I1MXVXZlpzdlpQUHpjCmIwTWhRamZvSTF3cHZMNk9YUlRv
-            U2tOK3E5MFBFNERsUHVzVnhsUDFRd00KLS0tIFd1MUpaaFU0bWdVRjJ6NjFwcFZt
-            bkJGWFFWanFBK1drZlBNcHo0c3Bjc00K/vPBLocRhtcJ3snGYFr+H7qhbg6iSSPP
-            OSH8WnaM5JmmA9IQlm5uGiG74PHi5sg5d+bwG8pPQtMKN+Ndxh7JIA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETWpkcjhINktqeGxjdWxz
+            UTVVNC9kalorcVJOdHpJSkZJNXlGUHZ2VUdrCjRCclBTZnJEZ3JGOVpqS1Y0b0dt
+            eldFMS91WUc2Y1FnWWZoN0grc01pT0UKLS0tIC96TjlEaVBGRkZhZ0hac2lmbEdI
+            eHMzTFhsQ0FqY05uUEZSbExCcmdscEkKdxITlc0V5ayq+9fmj77SnEMFxKJhOOta
+            RfJhOQUv8g3nCN+SsuaOy0TitUCiDWh5XoB0DufEQPcS/kzGZN1Inw==
            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
+        - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlN291MzZOaU4zazhEeXBh
-            WlhoYmh2ZDBsZmc3cEthdW5paWpXbXQvUG1FCjBLZ0FPVWR3T2pVWTZrRmkxSWUr
-            MHhkUFFPK1Z0b2t1Z1J0VjlER1JvcGMKLS0tIE45YndxVW4vak1wcEJoZzhHQ0E0
-            NzA1cy80ZW5vUFplQzVMZ0txSmVkMUEKFUvgmJNdo9sV33gOx7LVUSCYvIqCNwaP
-            u+XoWTfg4kp9f4KVTy/8huPsVLhZBUaf6jI10mV2z4QwaLHje4JiHw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBydlQ4S1duQU53Wk1nd21K
+            d2RqM1F0VDFJVXB2aGRTZ2hxczI2V1lndVdrCjArVlE2N0RGZ0htUEZYdVlQMlU5
+            SWIwWHVCaWxaQTJMNzg3WC8xRS9IYzgKLS0tIDRvSS8ybVlrSy9zYjQ2NXBaMlZk
+            Ulg4cUFBejRoS3VEWkRaZEUxMExUeWMKNeq6TN1gaBNU9vAitGttcU+8HmFQipdm
+            LPwo4/toyf27emb4KGs0AV0Dm4Sxj9S3Xvrv1B+qvhfT638/RIUm2w==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-22T05:48:59Z"
-    mac: ENC[AES256_GCM,data:In/gSIYnXKbbv1lzS/nmSESCHBcBv/TtkvhzdNiIn73N4kP9aJ+1JE8Npix8zNItzk46DX+nHBk8Kwgl6uq26YtL+sMTBKh5K8Ny0H8ivlgS+olXswv3Y9h1cYD7FBHUKzbMuiJd0ppjC0ZIn20rRpb4d57rwUbvY0KstyQW4JA=,iv:DcdTAimbXXpKhhiB9rriS75+XGNOCcScqi/804+Xx6g=,tag:NHW+UViRmbUDHb0gTd9TDg==,type:str]
+        - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4YXpyOXE3MFovWEQvMVRr
+            TGVST3U0N2dCVDJGT1A3eUtlRis3bFEvTHlFClZHQ2xRWklMMCtER01QNEVHaVYr
+            MC94V3R4MVdNdUU3eXQ2RGFFVGo4VFEKLS0tIDQ4b2ZuMy9URUswWUZqNHlxandU
+            OFducVVzdGZGY0tnbFFBZDdjVzVkaUEKN8qAbbrd4pAHRGIN8O64fl7bQ6hx6Isr
+            Qx0xKeuhJCVXgtE8xc7xmnEhqrcONlflJ/XUnYV9jOkB71zSBJxruA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzczdPMDdWU1ZtckJRQm5j
+            UWJub0Yzd3NzOEh4YWdId01nYWI1YVY3dng0ClpEYXBJV2cvWEdjdXcwUFI3Y0NG
+            MDgvTmNZOXRQQndyVmRHamNRbzVaVU0KLS0tIGFKVTI4TkE2UjhDUSsxQTlNQ0Vk
+            QmFMNnlqbnhScC90T012K1QxRnRUOHcKAV7NxUn0CMcjKwK8zrocoLO1P9jc22uG
+            eG+vdJ6xzA99UX51aPxQOeEJgdFPEd3y1QJszQmRzThvid7y4lv0Cw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsVmpzenRvWE5EK2wzRFkx
+            SERZV0s1Rkt0ZnZ1U3JQSFNhdGVvaWhWcTA4CjVxK0Z0MHI0ZnMrUS9YYWhTTG1z
+            L2lVS1Q2UkVQd2x5b1E1eWpQVGp2ZHMKLS0tIHNLOGhTYjkzWkFEM05wYkRZeXFQ
+            SXNTSGZZSFE2bFhybXdIc1FUb1ZBd0kKkYzflPRk6GrE6t9oVGOzc8xcyZDxiIw8
+            9SVXIgV0WVpY4lnFKYKH2i4+1sIm6tKOpizlQxTg5VgmmrTtfazWAA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0NHpkOTFHaXRhVGNua0dV
+            alRieWJ6WG5ZNzlvcTR2aTVUeWFBVGVVUUNZCnY2VUZUOWVlNGY1ZldyVGE2bkpi
+            VXVtQ3IyK0kyV1cyMU5nN1lYaW1oOUkKLS0tIFRVRGFCNWlGendSVEhHY0w0QTl6
+            emJEQkQ3QlU0TFVWaW1uQytaUndmQlEKKahqJpX8vI+PASOzzod/sFvXSkQFnJ9O
+            YmnmiFxm5WZDPLHwkgVx8FgCq9RfAad4HybhsMjYPKXJ/fNa/WVZRA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-30T06:31:42Z"
+    mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1
--- a/machines/sops.nix
+++ b/machines/sops.nix
@ -10,20 +10,9 @@
      # TODO: How to generate this key when bootstrap?
      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
      secrets = {
-        github_public_token = {
-          owner = "root";
-        };
-        singbox_sg_server = {
-          owner = "root";
-        };
-        singbox_jp_server = {
-          owner = "root";
-        };
-        private_dns_address = {
-          owner = "root";
-        };
+        "prometheus/metrics_username" = { };
+        "prometheus/metrics_password" = { };
      };
-      secrets.grafana_cloud_api = lib.mkIf config.services.prometheus.enable { owner = "prometheus"; };
    };
  };
 }
--- a/machines/thorite/default.nix
+++ b/machines/thorite/default.nix
@ -30,7 +30,6 @@

    commonSettings = {
      auth.enable = true;
-      autoupgrade.enable = true;
    };

    nixpkgs.system = "x86_64-linux";
--- a/machines/thorite/monitoring.nix
+++ b/machines/thorite/monitoring.nix
@ -14,6 +14,8 @@ with my-lib;

    custom.monitoring = {
      grafana.enable = true;
+      loki.enable = true;
+      promtail.enable = true;
    };

    services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =
@ -41,6 +43,7 @@ with my-lib;
          "45.142.178.32:22"
          "home.xinyang.life:8000"
        ];
+        passwordFile = config.sops.secrets."prometheus/metrics_password".path;
      in
      (mkScrapes [
        {
@ -50,10 +53,12 @@ with my-lib;
          port = 8082;
        }
        {
+          inherit passwordFile;
          name = "gotosocial";
          address = "xinyang.life";
        }
        {
+          inherit passwordFile;
          name = "miniflux";
          address = "rss.xinyang.life";
        }
--- a/machines/weilite/default.nix
+++ b/machines/weilite/default.nix
@ -18,7 +18,6 @@
      auth.enable = true;
      nix = {
        enable = true;
-        enableMirrors = true;
      };
    };

--- a/modules/nixos/common-settings/mainland.nix
+++ b/modules/nixos/common-settings/mainland.nix
@ -0,0 +1,38 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    mkDefault
+    ;
+
+  cfg = config.inMainland;
+in
+{
+  options.inMainland = mkOption {
+    type = types.bool;
+    default = config.time.timeZone == "Asia/Shanghai";
+  };
+
+  config = mkIf cfg.enable {
+    nix.conf.extra-substituters = [
+      "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
+    ];
+
+    networking.timeServers = [
+      "cn.ntp.org.cn"
+      "ntp.ntsc.ac.cn"
+    ];
+
+    services.dae = {
+      enable = mkDefault true;
+    };
+  };
+}
--- a/modules/nixos/common-settings/nix-conf.nix
+++ b/modules/nixos/common-settings/nix-conf.nix
@ -21,7 +21,6 @@ in
      default = true;
      type = types.bool;
    };
-    enableMirrors = mkEnableOption "cache.nixos.org mirrors in Mainland China";
    signing = {
      enable = mkEnableOption "Sign locally-built paths";
      keyFile = mkOption {
@ -55,10 +54,6 @@ in
        "https://cache.garnix.io"
      ];

-      extra-substituters = mkIf cfg.enableMirrors [
-        "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
-      ];
-
      trusted-public-keys = [
        "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
        "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g="
--- a/modules/nixos/common-settings/proxy-server.nix
+++ b/modules/nixos/common-settings/proxy-server.nix
@ -9,8 +9,6 @@ let
    mkIf
    mkEnableOption
    mkOption
-    mkDefault
-    types
    ;

  cfg = config.commonSettings.proxyServer;
@ -26,6 +24,9 @@ let
  mkSingConfig =
    { uuid, password, ... }:
    {
+      log = {
+        level = "warn";
+      };
      inbounds =
        [
          {
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -4,10 +4,11 @@
    ./common-settings/autoupgrade.nix
    ./common-settings/nix-conf.nix
    ./common-settings/proxy-server.nix
+    ./common-settings/mainland.nix
    ./disk-partitions
    ./restic.nix
    ./vaultwarden.nix
-    ./prometheus
+    ./monitor
    ./hedgedoc.nix
    ./sing-box.nix
    ./kanidm-client.nix
--- a/modules/nixos/monitor/default.nix
+++ b/modules/nixos/monitor/default.nix
@ -0,0 +1,172 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    mkIf
+    mkMerge
+    types
+    ;
+  cfg = config.custom.prometheus;
+
+  mkRulesOption = mkOption {
+    type = types.listOf (
+      types.submodule {
+        options = {
+          name = mkOption { type = lib.types.str; };
+          rules = mkOption { type = lib.types.listOf lib.types.attrs; };
+        };
+      }
+    );
+  };
+in
+{
+  imports = [
+    ./exporters.nix
+    ./grafana.nix
+    ./loki.nix
+  ];
+
+  options = {
+    custom.monitoring = {
+      grafana = {
+        enable = mkEnableOption "grafana with oauth only";
+      };
+    };
+    custom.prometheus = {
+      enable = mkEnableOption "Prometheus instance";
+      ruleModules = mkRulesOption;
+      exporters = {
+        enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
+        node = {
+          enable = mkEnableOption "node exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+        blackbox = {
+          enable = mkEnableOption "blackbox exporter";
+          listenAddress = mkOption {
+            type = types.str;
+            default = "${config.networking.hostName}.coho-tet.ts.net";
+          };
+        };
+      };
+    };
+  };
+
+  config = mkMerge [
+    {
+      sops.secrets = {
+        "prometheus/metrics_username" = {
+          sopsFile = ../../../machines/secrets.yaml;
+          group = "prometheus-auth";
+          mode = "0440";
+        };
+
+        "prometheus/metrics_password" = {
+          sopsFile = ../../../machines/secrets.yaml;
+          group = "prometheus-auth";
+          mode = "0440";
+        };
+      };
+
+      users.groups.prometheus-auth.members = [
+        "prometheus"
+      ];
+    }
+    (mkIf cfg.enable {
+
+      services.tailscale = {
+        enable = true;
+        permitCertUid = config.services.caddy.user;
+        openFirewall = true;
+      };
+
+      services.caddy = {
+        enable = true;
+        virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
+          reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
+        '';
+      };
+      services.prometheus = mkIf cfg.enable {
+        enable = true;
+        port = 9091;
+        globalConfig.external_labels = {
+          hostname = config.networking.hostName;
+        };
+
+        scrapeConfigs = [
+          {
+            job_name = "prometheus";
+            static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
+          }
+        ];
+
+        alertmanager = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          logLevel = "debug";
+          configuration = {
+            route = {
+              receiver = "ntfy";
+            };
+            receivers = [
+              {
+                name = "ntfy";
+                webhook_configs = [
+                  {
+                    url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
+                      Alert {{.status}}
+                      {{range .alerts}}-----{{range $k,$v := .labels}}
+                      {{$k}}={{$v}}{{end}}
+                      {{end}}
+                    ''}";
+                    send_resolved = true;
+                  }
+                ];
+              }
+            ];
+          };
+        };
+
+        alertmanagers = [
+          {
+            scheme = "http";
+            static_configs = [
+              {
+                targets = [
+                  "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
+                ];
+              }
+            ];
+          }
+        ];
+        rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
+      };
+      custom.prometheus.ruleModules = [
+        {
+          name = "prometheus_alerts";
+          rules = [
+            {
+              alert = "JobDown";
+              expr = "up == 0";
+              for = "1m";
+              labels = {
+                severity = "critical";
+              };
+              annotations = {
+                summary = "Job {{ $labels.job }} down for 1m.";
+              };
+            }
+          ];
+        }
+      ];
+    })
+  ];
+}
--- a/modules/nixos/monitor/exporters.nix
+++ b/modules/nixos/monitor/exporters.nix
@ -0,0 +1,102 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkIf;
+  cfg = config.custom.prometheus.exporters;
+in
+{
+  config = {
+    systemd.services.tailscaled.after =
+      (lib.optional cfg.node.enable "prometheus-node-exporters.service")
+      ++ (lib.optional cfg.blackbox.enable "prometheus-blackbox-exporters.service")
+      ++ (lib.optional config.services.caddy.enable "caddy.service");
+
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      enabledCollectors = [
+        "loadavg"
+        "time"
+        "systemd"
+      ];
+      listenAddress = cfg.node.listenAddress;
+      port = 9100;
+    };
+
+    services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
+      enable = true;
+      listenAddress = cfg.blackbox.listenAddress;
+      configFile = pkgs.writeText "blackbox.config.yaml" (
+        lib.generators.toYAML { } {
+          modules = {
+            tcp4_connect = {
+              prober = "tcp";
+              tcp = {
+                ip_protocol_fallback = false;
+                preferred_ip_protocol = "ip4";
+                tls = false;
+              };
+              timeout = "15s";
+            };
+          };
+        }
+      );
+    };
+
+    # gotosocial
+    sops.templates."gotosocial_metrics.env" = {
+      content = ''
+        GTS_METRICS_AUTH_ENABLED=true
+        GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
+        GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
+      '';
+      group = "prometheus-auth";
+      mode = "0440";
+    };
+    systemd.services.gotosocial.serviceConfig = {
+      EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ];
+      SupplementaryGroups = [ "prometheus-auth" ];
+    };
+
+    services.gotosocial.settings = {
+      metrics-enabled = true;
+    };
+
+    services.immich.environment = {
+      IMMICH_TELEMETRY_INCLUDE = "all";
+    };
+
+    services.restic.server.prometheus = true;
+
+    # miniflux 
+    sops.templates."miniflux_metrics_env" = {
+      content = ''
+        METRICS_COLLECTOR=1
+        LOG_LEVEL=debug
+        METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
+        METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
+      '';
+      group = "prometheus-auth";
+      mode = "0440";
+    };
+
+    systemd.services.miniflux.serviceConfig = {
+      EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ];
+      SupplementaryGroups = [ "prometheus-auth" ];
+    };
+
+    services.ntfy-sh.settings.enable-metrics = true;
+
+    services.caddy.globalConfig = ''
+      servers {
+        metrics
+      }
+
+      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
+      }
+    '';
+  };
+}
--- a/modules/nixos/prometheus/grafana.nix
+++ b/modules/nixos/prometheus/grafana.nix
--- a/modules/nixos/monitor/loki.nix
+++ b/modules/nixos/monitor/loki.nix
@ -0,0 +1,166 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkMerge
+    ;
+  cfg = config.custom.monitoring;
+  port-loki = 3100;
+in
+{
+  options = {
+    custom.monitoring = {
+      loki.enable = mkEnableOption "loki";
+      promtail.enable = mkEnableOption "promtail";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.loki.enable {
+      services.loki = {
+        enable = true;
+        configuration = {
+          auth_enabled = false;
+          server.http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+          server.http_listen_port = port-loki;
+
+          common = {
+            ring = {
+              instance_addr = "${config.networking.hostName}.coho-tet.ts.net";
+              kvstore.store = "inmemory";
+            };
+            replication_factor = 1;
+            path_prefix = "/var/lib/loki";
+          };
+
+          schema_config.configs = [
+            {
+              from = "2024-12-01";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+
+          storage_config = {
+            filesystem.directory = "/var/lib/loki/chunks";
+          };
+
+          limits_config = {
+            reject_old_samples = true;
+            reject_old_samples_max_age = "168h";
+            allow_structured_metadata = false;
+          };
+        };
+      };
+    })
+    (mkIf cfg.promtail.enable {
+      services.promtail = {
+        enable = true;
+        configuration = {
+
+          server = {
+            http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+            http_listen_port = 28183;
+            grpc_listen_port = 0;
+          };
+
+          positions.filename = "/tmp/positions.yml";
+
+          clients = [
+            {
+              url = "http://thorite.coho-tet.ts.net:${toString port-loki}/loki/api/v1/push";
+            }
+          ];
+
+          scrape_configs = [
+            {
+              job_name = "journal";
+              # Copied from Mic92's config
+              journal = {
+                max_age = "12h";
+                json = true;
+                labels.job = "systemd-journal";
+              };
+              pipeline_stages = [
+                {
+                  json.expressions = {
+                    transport = "_TRANSPORT";
+                    unit = "_SYSTEMD_UNIT";
+                    msg = "MESSAGE";
+                    coredump_cgroup = "COREDUMP_CGROUP";
+                    coredump_exe = "COREDUMP_EXE";
+                    coredump_cmdline = "COREDUMP_CMDLINE";
+                    coredump_uid = "COREDUMP_UID";
+                    coredump_gid = "COREDUMP_GID";
+                  };
+                }
+                {
+                  # Set the unit (defaulting to the transport like audit and kernel)
+                  template = {
+                    source = "unit";
+                    template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
+                  };
+                }
+                {
+                  regex = {
+                    expression = "(?P<coredump_unit>[^/]+)$";
+                    source = "coredump_cgroup";
+                  };
+                }
+                {
+                  template = {
+                    source = "msg";
+                    # FIXME would be cleaner to have this in a match block, but could not get it to work
+                    template = "{{if .coredump_exe}}{{.coredump_exe}} core dumped (user: {{.coredump_uid}}/{{.coredump_gid}}, command: {{.coredump_cmdline}}){{else}}{{.msg}}{{end}}";
+                  };
+                }
+                { labels.coredump_unit = "coredump_unit"; }
+                {
+                  # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
+                  replace = {
+                    source = "unit";
+                    expression = "^(session-\\d+.scope)$";
+                    replace = "session.scope";
+                  };
+                }
+                { labels.unit = "unit"; }
+                {
+                  # Write the proper message instead of JSON
+                  output.source = "msg";
+                }
+                # silence nscd:
+                # ignore random portscans on the internet
+                { drop.expression = "refused connection: IN="; }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__journal__hostname" ];
+                  target_label = "host";
+                }
+              ];
+            }
+            # {
+            #   job_name = "caddy-access";
+            #   file_sd_configs = {
+            #     files = [
+            #       "/var/log/caddy/*.log"
+            #     ];
+            #     refresh_interval = "5m";
+            #   };
+            # }
+          ];
+        };
+      };
+    })
+  ];
+}
--- a/modules/nixos/prometheus/restic.nix
+++ b/modules/nixos/prometheus/restic.nix
--- a/modules/nixos/prometheus/default.nix
+++ b/modules/nixos/prometheus/default.nix
@ -1,148 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib)
-    mkEnableOption
-    mkOption
-    mkIf
-    types
-    ;
-  cfg = config.custom.prometheus;
-
-  mkRulesOption = mkOption {
-    type = types.listOf (
-      types.submodule {
-        options = {
-          name = mkOption { type = lib.types.str; };
-          rules = mkOption { type = lib.types.listOf lib.types.attrs; };
-        };
-      }
-    );
-  };
-in
-{
-  imports = [
-    ./exporters.nix
-    ./grafana.nix
-  ];
-
-  options = {
-    custom.monitoring = {
-      grafana = {
-        enable = mkEnableOption "grafana with oauth only";
-      };
-    };
-    custom.prometheus = {
-      enable = mkEnableOption "Prometheus instance";
-      ruleModules = mkRulesOption;
-      exporters = {
-        enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
-        node = {
-          enable = mkEnableOption "node exporter";
-          listenAddress = mkOption {
-            type = types.str;
-            default = "${config.networking.hostName}.coho-tet.ts.net";
-          };
-        };
-        blackbox = {
-          enable = mkEnableOption "blackbox exporter";
-          listenAddress = mkOption {
-            type = types.str;
-            default = "${config.networking.hostName}.coho-tet.ts.net";
-          };
-        };
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.tailscale = {
-      enable = true;
-      permitCertUid = config.services.caddy.user;
-      openFirewall = true;
-    };
-
-    services.caddy = {
-      enable = true;
-      virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
-        reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
-      '';
-    };
-    services.prometheus = mkIf cfg.enable {
-      enable = true;
-      port = 9091;
-      globalConfig.external_labels = {
-        hostname = config.networking.hostName;
-      };
-
-      scrapeConfigs = [
-        {
-          job_name = "prometheus";
-          static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
-        }
-      ];
-
-      alertmanager = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        logLevel = "debug";
-        configuration = {
-          route = {
-            receiver = "ntfy";
-          };
-          receivers = [
-            {
-              name = "ntfy";
-              webhook_configs = [
-                {
-                  url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
-                    Alert {{.status}}
-                    {{range .alerts}}-----{{range $k,$v := .labels}}
-                    {{$k}}={{$v}}{{end}}
-                    {{end}}
-                  ''}";
-                  send_resolved = true;
-                }
-              ];
-            }
-          ];
-        };
-      };
-
-      alertmanagers = [
-        {
-          scheme = "http";
-          static_configs = [
-            {
-              targets = [
-                "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
-              ];
-            }
-          ];
-        }
-      ];
-      rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
-    };
-    custom.prometheus.ruleModules = [
-      {
-        name = "prometheus_alerts";
-        rules = [
-          {
-            alert = "JobDown";
-            expr = "up == 0";
-            for = "1m";
-            labels = {
-              severity = "critical";
-            };
-            annotations = {
-              summary = "Job {{ $labels.job }} down for 1m.";
-            };
-          }
-        ];
-      }
-    ];
-  };
-}
--- a/modules/nixos/prometheus/exporters.nix
+++ b/modules/nixos/prometheus/exporters.nix
@ -1,65 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkIf;
-  cfg = config.custom.prometheus.exporters;
-in
-{
-  config = {
-    services.prometheus.exporters.node = mkIf cfg.node.enable {
-      enable = true;
-      enabledCollectors = [
-        "loadavg"
-        "time"
-        "systemd"
-      ];
-      listenAddress = cfg.node.listenAddress;
-      port = 9100;
-    };
-
-    services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
-      enable = true;
-      listenAddress = cfg.blackbox.listenAddress;
-      configFile = pkgs.writeText "blackbox.config.yaml" (
-        lib.generators.toYAML { } {
-          modules = {
-            tcp4_connect = {
-              prober = "tcp";
-              tcp = {
-                ip_protocol_fallback = false;
-                preferred_ip_protocol = "ip4";
-                tls = false;
-              };
-              timeout = "15s";
-            };
-          };
-        }
-      );
-    };
-
-    services.gotosocial.settings = {
-      metrics-enabled = true;
-    };
-
-    services.immich.environment = {
-      IMMICH_TELEMETRY_INCLUDE = "all";
-    };
-
-    services.restic.server.prometheus = true;
-    systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
-    services.ntfy-sh.settings.enable-metrics = true;
-
-    services.caddy.globalConfig = ''
-      servers {
-        metrics
-      }
-
-      admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
-      }
-    '';
-  };
-}
--- a/overlays/my-lib/prometheus.nix
+++ b/overlays/my-lib/prometheus.nix
@ -7,6 +7,7 @@ in
    {
      name,
      address,
+      passwordFile ? null,
      port ? 443,
      scheme ? "https",
      ...
@ -16,6 +17,15 @@ in
      scheme = scheme;
      static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
    }
+    // (
+      if isNull passwordFile then
+        { }
+      else
+        {
+          basic_auth.username = "prom";
+          basic_auth.password_file = passwordFile;
+        }
+    )
  );

  mkCaddyScrapes = mkFunction (
Author	SHA1	Message	Date
xinyangli	2ec4b611a8	modules/monitoring: add loki and promtail	2024-12-02 14:44:26 +08:00
xinyangli	92db38383e	modules/prometheus: fix basic auth scrape	2024-12-02 10:43:43 +08:00
xinyangli	74b67e1854	modules/prometheus: add basic auth	2024-11-30 17:35:46 +08:00
xinyangli	97fcdefc2b	modules/prometheus: move to monitor directory	2024-11-30 12:56:54 +08:00
xinyangli	9e3af9a535	modules/prometheus: start exporters after tailscaled	2024-11-30 12:54:51 +08:00