modules/monitoring: add loki and promtail

.sops.yaml

@@ -18,6 +18,9 @@ creation_rules:
       - *host-massicot
       - *host-thorite
       - *host-biotite
+      - *host-hk-00
+      - *host-fra-00
+      - *host-la-00
   - path_regex: machines/calcite/secrets.yaml
     key_groups:
     - age:
@@ -33,7 +36,7 @@ creation_rules:
     - age:
       - *xin
       - *host-massicot
-  - paht_regex: machines/biotite/secrets.yaml
+  - path_regex: machines/biotite/secrets.yaml
     key_groups:
     - age:
       - *xin
@@ -45,11 +48,11 @@ creation_rules:
       - *host-thorite
   - path_regex: machines/dolomite/secrets/secrets.yaml
     key_groups:
-    - age:
+    - age: 
       - *xin
-      - *host-la-00
       - *host-hk-00
       - *host-fra-00
+      - *host-la-00
   - path_regex: machines/dolomite/secrets/la-00.yaml
     key_groups:
     - age:
@@ -60,7 +63,6 @@ creation_rules:
     - age:
       - *xin
       - *host-hk-00
-
   - path_regex: machines/dolomite/secrets/fra-00.yaml
     key_groups:
     - age:

machines/biotite/default.nix

@@ -1,12 +1,13 @@
 {
-  config,
   lib,
-  pkgs,
   ...
 }:
 
 {
-  imports = [ ./hardware-configurations.nix ];
+  imports = [
+    ./hardware-configurations.nix
+    ./services/gotosocial.nix
+  ];
 
   networking.hostName = "biotite";
   networking.useNetworkd = true;
@@ -20,11 +21,28 @@
     address = [ "2a03:4000:4a:148::1/64" ];
   };
 
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   commonSettings = {
     auth.enable = true;
     autoupgrade.enable = true;
   };
 
+  custom.monitoring = {
+    promtail.enable = true;
+  };
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+  };
+
+  services.caddy.enable = true;
+  services.tailscale.enable = true;
+
   users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
 
   system.stateVersion = "24.11";

machines/biotite/secrets.yaml

@@ -0,0 +1,31 @@
+gotosocial:
+    oidc_client_secret: ENC[AES256_GCM,data:KVQxzs67sohax2h0Y/jjhnbY4fetrdVvWhBGbqgDSGgBC7QazrOmTA++BSRzMmVv,iv:HIRMc56aLanqQRTWH9E0wzzXymImi0pxK/ccPEP8Fcc=,tag:PMhOLeE3mKIIQveRdfpgpA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxVXpUNXA3eEZEeGxpMmZT
+            L0lPUzYzNXlrS2JDbWlYNzJiYmwwYm1PSjFNCjAzSGluME1hd1Fnc0ZCNUhUMzdU
+            UHkwbmxwdTdVOFhIYUo3N0laVlJRV0EKLS0tIHR5NDJqQnI3ZkFGcmwwaHZwOGd2
+            Y2gvVTRMc2RSd1UxWUdEWVZDRm5VbHMKLYJ59s2MDDokJRAAXoTAL1VTU4WKY8qS
+            GiXZu954JzacAR9Ey2GQTFdMN73Aw+PbiWw6cph33gZaOQt9/QA92w==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3djErT0VVOU9ydmpjL01a
+            aDFQa2JiMVBURzhCZ0NBUDdaMDZCV2piUjI0ClBmSGJIallnTzdmV3RYZlNBK0Ji
+            K21qRkg0SDY3WkZ5bXFrWitBSGNEQ1EKLS0tIGhHMGRsZGNaL2hNWFdKUTJUUk1G
+            RzBMVDNjS29SUkdRK3dIV01sU0hYR3cK1SbvKAM6Gpsffv3HIi/WtWnCZUBic0AT
+            ZRv4pvJBx1oxWsKIHW0t6VrqWMQ+suup8p6dW+h5HE8Z4ciIMrXLEg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-12-02T05:10:32Z"
+    mac: ENC[AES256_GCM,data:ZAdFsjVuk1Fiv+DKmHrc1yu1XQpRDmRHaQhu5hduSZUa1W1cXdTlChvIW5vADFg5tVCjuYptuLvCMW+ZSQeqqG2ntHHZ+IkuovZzKFuc+BIiL/jF2ZzbyJ7X4Wj1GziCScHVxx98dgbpFoufHe6N3wCaHmngo1RYsY5N1RRbRdU=,iv:5IMQ0kOX9UAOm8bcsQRyu6zu8GJjvnHFufCNjY0s9UI=,tag:zBEPSR9DZDpwbCaIka8mXA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.1

machines/biotite/services/gotosocial.nix

@@ -0,0 +1,46 @@
+{ config, ... }:
+{
+  sops.secrets."gotosocial/oidc_client_secret" = {
+    owner = "gotosocial";
+  };
+
+  sops.templates."gotosocial.env" = {
+    owner = "gotosocial";
+    content = ''
+      GTS_OIDC_CLIENT_SECRET=${config.sops.placeholder."gotosocial/oidc_client_secret"}
+    '';
+  };
+
+  services.gotosocial = {
+    enable = true;
+    settings = {
+      log-level = "info";
+      bind-address = "127.0.0.1";
+      port = 19571;
+      host = "gts.xiny.li";
+      account-domain = "xiny.li";
+      letsencrypt-enabled = false;
+      instance-expose-public-timeline = true;
+      oidc-enabled = true;
+      oidc-idp-name = "Kanidm";
+      oidc-issuer = "https://auth.xinyang.life/oauth2/openid/gotosocial";
+      oidc-client-id = "gotosocial";
+      oidc-link-existing = true;
+    };
+    environmentFile = config.sops.templates."gotosocial.env".path;
+  };
+
+  services.caddy = {
+    virtualHosts."https://gts.xiny.li".extraConfig = ''
+      encode zstd gzip
+      reverse_proxy * http://${config.services.gotosocial.settings.bind-address}:${toString config.services.gotosocial.settings.port} {
+          flush_interval -1
+      }
+    '';
+    virtualHosts."https://xiny.li".extraConfig = ''
+      redir /.well-known/host-meta* https://gts.xiny.li{uri} permanent  # host
+      redir /.well-known/webfinger* https://gts.xiny.li{uri} permanent  # host
+      redir /.well-known/nodeinfo* https://gts.xiny.li{uri} permanent   # host
+    '';
+  };
+}

machines/calcite/configuration.nix

@@ -18,7 +18,6 @@ in
   commonSettings = {
     auth.enable = true;
     nix = {
-      enableMirrors = true;
       signing.enable = true;
     };
   };
@@ -330,10 +329,6 @@ in
   system.switch.enable = false;
   system.switch.enableNg = true;
 
-  nix.extraOptions = ''
-    !include "${config.sops.secrets.github_public_token.path}"
-  '';
-
   sops.secrets = {
     "restic/repo_url" = {
       owner = "xin";

machines/dolomite/common.nix

@@ -2,6 +2,7 @@
 {
   config = {
     sops = {
+      age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
       secrets = {
         wg_private_key = {
           owner = "root";
@@ -33,6 +34,10 @@
       node.enable = true;
     };
 
+    custom.monitoring = {
+      promtail.enable = true;
+    };
+
     services.tailscale.enable = true;
 
     commonSettings = {

machines/dolomite/fra.nix

@@ -62,7 +62,5 @@
     address = [ "185.217.108.59/24" ];
   };
 
-  custom.prometheus.enable = false;
-
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
 }

machines/dolomite/secrets/secrets.yaml

@@ -1,6 +1,6 @@
 sing-box:
-    password: ENC[AES256_GCM,data:aifvj/rBvmIF6M4SJ6j4rkw0J0oBGUmO,iv:C9KlVngh74z/VjjOGxnlpA4CqFv7TCSD3KSm2l/xGB4=,tag:10zUgbP2exTQ4KK0zeMM2A==,type:str]
+    password: ENC[AES256_GCM,data:qCc1v8nAL0oYisRinMDXGrBQA+r6XNoa,iv:eTxtad4kEdE28XqnrZEek8BtXNY1rNgLvGLxlMzRtl4=,tag:s/shWAkYE4DSnScpTY8ulQ==,type:str]
-    uuid: ENC[AES256_GCM,data:ZPEqllAXeLMyVEp/6+9LSL346J2tiuM5tYs404/vp9rnkrvc,iv:Oy/U1c2sW5a2eQQxXAEjqaE85xX5rFapz9k/DtcZR+w=,tag:BHU+ScDBeWnctkDBRnm+4g==,type:str]
+    uuid: ENC[AES256_GCM,data:lEpz15sLOVrGDzQwTJyS+tFJY0bMeO265bxocWAjB6qrvxYx,iv:lhk5jl/udUH3AZEuk5ffuvin/qhRUaOZ/3nk1Jaw+DI=,tag:4mKFIVKT+D47njfDsxe9iA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -10,50 +10,41 @@ sops:
         - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmdCtZK2FVRTh3YVd3dm9m
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxV0RYNXFMdlhqc0RhdHEx
-            ZWR5VVIvS3VOSGh2cmg2ZUFrYmNIdVNLSTNVCjlhVlJER1BZMlRUd1RkYnpvTE9F
+            bkNXT0tYYmpTK2NyUnpKRjQzVkpTMk4yVHlFClBaVHZoVXlqRXFxYStzR2U0MzVG
-            bExGa1NBWWR0enBmUFJYVVA4UlI1cUkKLS0tIC8wa3FGRnFldVdTdkpBb2xQc3BD
+            OHI0Qjl0amw0V2tneWtrUHpSYVg3VmMKLS0tIEpneDFuVWZ2TFUwN0QxZWJnVEE3
-            cTlhNHplRUoyS3pxNnF0TVlFTy9kdzQK4kDSzSV4ZnELvCsajGwvsc/vzua2hbI1
+            SEhGMG9ac3gyb21Sa3V0cnB5SnppM1EKzfuKBAjPChde2UAEib3yE5Dczv3/UePL
-            Vht7rmZ8Dl4Y3xEIXG7XVnWK2GOblpqZ/eza1T6kWEkXp2uCdQnM6Q==
+            rHHxxSr6kIPIwtcjJpJJxqndLSCegXaomZukxuble3Xt4Nl4sVhaFg==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1dVV0U3kwSmdnTU1HcGpr
-            U2FKZVV1c1R6a3ovRGxoOUlrcUNWUUFHN25ZClBBTUZGeTc0Tkx1OXdaK1p6aWpr
-            aSsvN0ZDR1V3VnVrb1FBYzdHSTNXOVkKLS0tIFlSUk5LT1hVUUd1aVg1eVNTUURX
-            OXRVVmNRWEhmVXZkWC9HNTUyUTNrMlUK370K3D1vU97vHV9aGjYrFOIJzmOQAnzH
-            QR6XsOkM0FRvSkhTsEZ3qC4Wd2MTIyRzHYPKvZmz9LufIr1N/JFj1Q==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQT0YyeXI4d2o4V0lWUE4x
-            ZXZWWDFiakdqNlU5RWt6QUdxYVRSZzQyZkZBCi9Tdm5wRXB2cTYxdnVYRXJaS0d0
-            Lzg3VWpqQ1NOb1NTYXE4RGVRZVZoM1UKLS0tIFdGM01VU3FEc0ZyeEN3bVM1WEZq
-            M3BFa1hoWkQyRkJqSlZiTnBwQWphemcKLTAza2y96h+IyWB2EN6e4WIFQqeL5E7p
-            CDmHr+hSt6u9cr8C/etljxGMbKf9GqFOeuCyPugrJGdu4/qlR5iE0g==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFa1RHN2s4ajYzZmwvUlN2
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1ci91a1lQa0NXUFhZNWEr
-            c05SdERTTEhPRnJWOUF6TExIMnBEZkVMb1I4CkxBeTRQWmZEOGNrcFlGV2wrMkhI
+            cHlsQ093NHpESGdZN2xvdEUyZS9SKzBvSkhvClBZNG82OGR1WTZUUXhCb0ZyYmVX
-            QnAwSzZPaWNWbmdnZmFjZVJyRVdzN2cKLS0tIHVMU3Z6a1MrV3BVV1hqbEdYODJu
+            UlZlaHNxL3Z0ZjZ2dVRoOEJibWVZR0EKLS0tIFpQMlQyaTY0bHVsUk9nekh4R2dK
-            cGgvNU05eGx4alRNT2d5MWp6Q3lWZDAKQ+D1niMzaso/lQwdmepvACF8/SDEt2mQ
+            YkhMdG9MbUpDZzJvcXE0bkpLeXZVT3cKLCgizqmjO1hueLvvAWVyZ9dPQcYOQHwW
-            7nTRVJIpjGPTxO4ezcQWUGej+BSEnOoZno3epoIXLNlwDnHOAawTWQ==
+            pE//uiFFpjRsXLVB556ZyGYHn4osTfq73XYqvpsE4gsxT2scGxP/ZQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNcHNReHZibVlrNUtncnl1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4VmdsdHJJc0pMOVFUSmV5
-            SzczRGVFdUNvcFdqeWpZUk5FL0hwOS9LT3l3CnFLdXozcUxXYUpjUXJZWEtjMXo3
+            MUJaNTVWZUVrd2RrMUQxYVA2UlRlallwSmxJCjVOQWZESnViZVMxTTZPMElocm1C
-            d28reWd0Z1Y0NWdBTG1MTkRGSEphY2sKLS0tIGw5U3NiOU1DNitUd0x5SkJ3SHFj
+            TGlsOW90UytISDlGQi9zaGlZQ3BPamcKLS0tIDQwMW9WbUl2c29sVWxSWUk3bHAr
-            RVpWNDNUb2d1SEZpQlFBK2tFVjFzU0kKtI7e+kkiBm1L/WzkBApRI8IIo3gHdrE1
+            R0tTMHlPUlgxNVg0YlFyMm1kSm9ReHcKCMO2+wSj5OQJ+ClRsPADL9Zfg7oN6AzJ
-            fzR+sbYEHWf95iEmb/oGlH++TrFW/zRXEyWPAi4ORTs7s/Ql1UC4Wg==
+            IgKibbO2MGx/S+6x5K/QGEvaFWqh6bAWDgvdq/9I1kaO+fMpsmMqCA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-22T05:51:19Z"
+        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
-    mac: ENC[AES256_GCM,data:LPUb7YbELPsgYX+LvfuGdiNG1B5ZrvyRVZL9UiMHoJMDHaWpDGCQkT1bk5jEOewwFh+StK560UsPK4uW0+SqADJO80rmM3xfqlp5Ul2HJ8mU19l5C6FLpv2REIzhCp333rNJJlyhn3H6GZgMaWnjjLSX9XMOVaq1iz6Qt0P76SI=,iv://EdDr5D51RFuucq8gkei2RC7H2bkRYxP/7lerp9dtk=,tag:JrRQ19sZ0OAZouMgOiU/tQ==,type:str]
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxR0tzQ2JuZkRrMGlWN0JJ
+            U3pwdHBmQ1N2NUlyT0s4REpmVFEyRk9XeFdrCkZQdXRPMktjYnZqc0trOGtNeHd1
+            QjZXZlozaVhYRUZ1TzQ0QVRxb20xZEUKLS0tIEF4WVh6VTFVVVVuajlXUGRSS2tS
+            K1F1SzI2NFNIKzlreVBXSjAxaUxQd28KFaf1uu7OlqIe0TirJFgS3iPjhXPyfNDE
+            m2XUjzdXp+chJCzVOFvpYStqz+e08ADEc+jp3YsTLcxyqvXhQdyL/Q==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-12-02T05:26:17Z"
+    mac: ENC[AES256_GCM,data:K94zFWPWGUisLCqDjSLs17QxHXPH4tPU/98Sb4lCnt7IRAIn14x/T+BnInY/DK+DOVLLtzSfuN0kgzzGjSzwJx5Vq1G3MkhngRQQRT9dvODTCMAw6lPt98Ofw1CEEsFQnpYo9zIUlCGKg2YPKFLqE7OjkPxqw7VYvgzr5dDw58s=,iv:3xcJfNX5v/e9HgZt3UrHs2/C5ivaBV1rXKIBs9hKKFg=,tag:RQPQQ1cmZiOpQjUwqnzZQA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.1

machines/massicot/services.nix

@@ -43,6 +43,10 @@ in
     environmentFile = config.sops.secrets.hedgedoc_env.path;
   };
 
+  custom.monitoring = {
+    promtail.enable = true;
+  };
+
   custom.prometheus.exporters = {
     enable = true;
     blackbox = {

machines/osmium/default.nix

@@ -51,7 +51,7 @@
     };
 
     commonSettings = {
-      nix.enableMirrors = true;
+      nix.enable = true;
       auth.enable = true;
     };
 

machines/raspite/configuration.nix

@@ -9,7 +9,7 @@
   imports = [ ./hass.nix ];
 
   commonSettings = {
-    nix.enableMirrors = true;
+    nix.enable = true;
     auth.enable = true;
   };
 

machines/secrets.yaml

@@ -10,47 +10,74 @@ sops:
         - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmUUtKcU0va1F3SjFzcktl
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwMHB1bFQ3dWJIU3NiOVVP
-            S0FJNHlTUzBkMWlydGRjM1Y4SGd2SXpMVTJRCmtRSW9wNW9xMDBaQ0YzUWM4YjRz
+            Yi9LZE1PTVdMY1BqS1JHV3VPLzZIY0hGK0NZClNlclVXKzBvNTBrTlhiR0VsaVoz
-            SVRDNHRjNG5hTHBOOHorTTlJU1BwY1EKLS0tIEpLREJ1VzFaalczZlhKaitHVTJU
+            RlVLNVBEVDgzSXB5ZGxDd3hqNDh2V2MKLS0tIEhBZHFUY3c2VXJBVEVKamZ6TzBa
-            MDdJaVBtVmw4WTlBUEF5WXJSVFRFeDAKnvF6CmnU8hxXSdKQPUJqPT7Dewl4REOH
+            MlFsNnVEV0xCdlJoRnBhUHF2MmswUEUKNYD9zssGBy9SaKeOMvTz71B6KMPW87cM
-            wDQELRaDkMPMKEOAc6wCmXNErvj/I7w7wuvB5WxtanC7g4IEphD6aA==
+            tFJzgnQceEQF658lVa5cCzG1gzraCgBtQU15XzC7e8zWI9CHquRRlQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBS2xqbWp0NkFMVm1Cci82
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTTnZLTlZQRzc1enVEa1BN
-            d0ZmRXdRc2FFSklHNnZtV1Z0REFIelZDQ1U4CllmdllNVnp3WmpCeDNBRzVFbVR5
+            SHdoSi9oOXk4UTV0SlRZS2tLS2FFL3VjNzNNClVWTTNKekF6T0RTUzdEeWhLbHoz
-            WkJFMGs5ZWJEK1lSWDQwYUdOdFJseGcKLS0tIEZUU254aWtYdWthL3I2UkJ0eklj
+            WFZKaHJEaVBWa04zRWRiVnJZRjU0YVEKLS0tIFJVL0FEemowS3V6MmsxbWJMU2I1
-            WHhrRlRvLzlmY0REYktGSlh3MENzRzgKzO1XqXhcXAxfn86+IY+ccBII1SGYctAk
+            U2NnUnVKdFlRSGVzUFQ4ZFcwL0lWTlkKz1t3yqjgIdMWS/Nsy2nq3oCjOhGDP+UT
-            +ArpGmXaf53RFmPLSzMGNaiJzfhqk9U9bn3WV9CFdaA7Rtec0ZAcNw==
+            L+LAuFExJPV0qlsOG/kCGB/WtCJfnBvcp6vPDBLqjK8NllIX/iPI5g==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycmppSGR1YTMzTTJQSmx0
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETWpkcjhINktqeGxjdWxz
-            eFlRQWpPRTR1L042MXlqd3dURStjaUZjR1M4Ci9VdzZkSmN3d1BDNTlJdXNSenhZ
+            UTVVNC9kalorcVJOdHpJSkZJNXlGUHZ2VUdrCjRCclBTZnJEZ3JGOVpqS1Y0b0dt
-            MDE1VktBN252L2FYMjJmNEFITGptM3MKLS0tIDNYOEZqNjM1VCtEWDlGdzYveG5j
+            eldFMS91WUc2Y1FnWWZoN0grc01pT0UKLS0tIC96TjlEaVBGRkZhZ0hac2lmbEdI
-            dEp2bmVQMmV5ZU9Jb3FFSDFoT2NJOE0Kx1ZifyU2WLoHeUmqP9oCUmIl6ZJeytGB
+            eHMzTFhsQ0FqY05uUEZSbExCcmdscEkKdxITlc0V5ayq+9fmj77SnEMFxKJhOOta
-            WPMJKcNtuJHL1OWhT0wMiv6NEF5UaYXIlCqSVtXAMy554G4JlX5tQw==
+            RfJhOQUv8g3nCN+SsuaOy0TitUCiDWh5XoB0DufEQPcS/kzGZN1Inw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5OGlRWVM0Z01pYjdqYjEy
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBydlQ4S1duQU53Wk1nd21K
-            TGdhNFhpNEUwSFZaeHNwRkpraFF5RTU2SXhnCm5YWmM5SmdERzZBWTgva1E4MDFm
+            d2RqM1F0VDFJVXB2aGRTZ2hxczI2V1lndVdrCjArVlE2N0RGZ0htUEZYdVlQMlU5
-            N0xyUExGV0MvbFF3M0ZRSVEydFNUSGMKLS0tIGxxNWhsSEt4WDR2a2hId1JkVFE3
+            SWIwWHVCaWxaQTJMNzg3WC8xRS9IYzgKLS0tIDRvSS8ybVlrSy9zYjQ2NXBaMlZk
-            enJ6MzJxR0I4eStSQk9ON0dsdjFmRkEKBSGkv1O0vgHSsU3+6AGN7bKQ5lpN7AMT
+            Ulg4cUFBejRoS3VEWkRaZEUxMExUeWMKNeq6TN1gaBNU9vAitGttcU+8HmFQipdm
-            eqEgWx7juZ7hKzLq1HMbiT61l0FrJNHEMfn15bzn7GsK5YJQvfiq9w==
+            LPwo4/toyf27emb4KGs0AV0Dm4Sxj9S3Xvrv1B+qvhfT638/RIUm2w==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeVlxRk1PZUpUVm1xckRu
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4YXpyOXE3MFovWEQvMVRr
-            ZmMzdHdvKzExaTQxVHYvYjIvblo3b1ZORFJrCkpVdHFLbCtNS0xnamJ0T250YzUy
+            TGVST3U0N2dCVDJGT1A3eUtlRis3bFEvTHlFClZHQ2xRWklMMCtER01QNEVHaVYr
-            Uy9Xd0tMa3FSVlRkQXFaTWJVem9uWGsKLS0tIFRmT0VzL0hlLzkrRTZxcWtLN3Qv
+            MC94V3R4MVdNdUU3eXQ2RGFFVGo4VFEKLS0tIDQ4b2ZuMy9URUswWUZqNHlxandU
-            YVMya3dUazFyaWRNNDJ3OVNIVXJLVTQK+7MxkmBjPszozXUO+zVaWdsovDmhWAfz
+            OFducVVzdGZGY0tnbFFBZDdjVzVkaUEKN8qAbbrd4pAHRGIN8O64fl7bQ6hx6Isr
-            8puIpXpWZY09BkS0vs4oNhiVA9PD11TBIVCEbC5E1TwpwboMXBYhCQ==
+            Qx0xKeuhJCVXgtE8xc7xmnEhqrcONlflJ/XUnYV9jOkB71zSBJxruA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzczdPMDdWU1ZtckJRQm5j
+            UWJub0Yzd3NzOEh4YWdId01nYWI1YVY3dng0ClpEYXBJV2cvWEdjdXcwUFI3Y0NG
+            MDgvTmNZOXRQQndyVmRHamNRbzVaVU0KLS0tIGFKVTI4TkE2UjhDUSsxQTlNQ0Vk
+            QmFMNnlqbnhScC90T012K1QxRnRUOHcKAV7NxUn0CMcjKwK8zrocoLO1P9jc22uG
+            eG+vdJ6xzA99UX51aPxQOeEJgdFPEd3y1QJszQmRzThvid7y4lv0Cw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsVmpzenRvWE5EK2wzRFkx
+            SERZV0s1Rkt0ZnZ1U3JQSFNhdGVvaWhWcTA4CjVxK0Z0MHI0ZnMrUS9YYWhTTG1z
+            L2lVS1Q2UkVQd2x5b1E1eWpQVGp2ZHMKLS0tIHNLOGhTYjkzWkFEM05wYkRZeXFQ
+            SXNTSGZZSFE2bFhybXdIc1FUb1ZBd0kKkYzflPRk6GrE6t9oVGOzc8xcyZDxiIw8
+            9SVXIgV0WVpY4lnFKYKH2i4+1sIm6tKOpizlQxTg5VgmmrTtfazWAA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0NHpkOTFHaXRhVGNua0dV
+            alRieWJ6WG5ZNzlvcTR2aTVUeWFBVGVVUUNZCnY2VUZUOWVlNGY1ZldyVGE2bkpi
+            VXVtQ3IyK0kyV1cyMU5nN1lYaW1oOUkKLS0tIFRVRGFCNWlGendSVEhHY0w0QTl6
+            emJEQkQ3QlU0TFVWaW1uQytaUndmQlEKKahqJpX8vI+PASOzzod/sFvXSkQFnJ9O
+            YmnmiFxm5WZDPLHwkgVx8FgCq9RfAad4HybhsMjYPKXJ/fNa/WVZRA==
             -----END AGE ENCRYPTED FILE-----
     lastmodified: "2024-11-30T06:31:42Z"
     mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str]

machines/thorite/monitoring.nix

@@ -14,6 +14,8 @@ with my-lib;
 
     custom.monitoring = {
       grafana.enable = true;
+      loki.enable = true;
+      promtail.enable = true;
     };
 
     services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =

machines/weilite/default.nix

@@ -18,7 +18,6 @@
       auth.enable = true;
       nix = {
         enable = true;
-        enableMirrors = true;
       };
     };
 

modules/nixos/common-settings/mainland.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    mkDefault
+    ;
+
+  cfg = config.inMainland;
+in
+{
+  options.inMainland = mkOption {
+    type = types.bool;
+    default = config.time.timeZone == "Asia/Shanghai";
+  };
+
+  config = mkIf cfg {
+    nix.settings.extra-substituters = [
+      "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
+    ];
+
+    networking.timeServers = [
+      "cn.ntp.org.cn"
+      "ntp.ntsc.ac.cn"
+    ];
+
+    services.dae = {
+      enable = mkDefault true;
+    };
+  };
+}

modules/nixos/common-settings/nix-conf.nix

@@ -21,7 +21,6 @@ in
       default = true;
       type = types.bool;
     };
-    enableMirrors = mkEnableOption "cache.nixos.org mirrors in Mainland China";
     signing = {
       enable = mkEnableOption "Sign locally-built paths";
       keyFile = mkOption {
@@ -55,10 +54,6 @@ in
         "https://cache.garnix.io"
       ];
 
-      extra-substituters = mkIf cfg.enableMirrors [
-        "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
-      ];
-
       trusted-public-keys = [
         "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
         "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g="

modules/nixos/common-settings/proxy-server.nix

@@ -9,8 +9,6 @@ let
     mkIf
     mkEnableOption
     mkOption
-    mkDefault
-    types
     ;
 
   cfg = config.commonSettings.proxyServer;
@@ -26,6 +24,9 @@ let
   mkSingConfig =
     { uuid, password, ... }:
     {
+      log = {
+        level = "warn";
+      };
       inbounds =
         [
           {

modules/nixos/default.nix

@@ -4,6 +4,7 @@
     ./common-settings/autoupgrade.nix
     ./common-settings/nix-conf.nix
     ./common-settings/proxy-server.nix
+    ./common-settings/mainland.nix
     ./disk-partitions
     ./restic.nix
     ./vaultwarden.nix

modules/nixos/monitor/default.nix

@@ -28,6 +28,7 @@ in
   imports = [
     ./exporters.nix
     ./grafana.nix
+    ./loki.nix
   ];
 
   options = {

modules/nixos/monitor/loki.nix

@@ -0,0 +1,166 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkMerge
+    ;
+  cfg = config.custom.monitoring;
+  port-loki = 3100;
+in
+{
+  options = {
+    custom.monitoring = {
+      loki.enable = mkEnableOption "loki";
+      promtail.enable = mkEnableOption "promtail";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.loki.enable {
+      services.loki = {
+        enable = true;
+        configuration = {
+          auth_enabled = false;
+          server.http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+          server.http_listen_port = port-loki;
+
+          common = {
+            ring = {
+              instance_addr = "${config.networking.hostName}.coho-tet.ts.net";
+              kvstore.store = "inmemory";
+            };
+            replication_factor = 1;
+            path_prefix = "/var/lib/loki";
+          };
+
+          schema_config.configs = [
+            {
+              from = "2024-12-01";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+
+          storage_config = {
+            filesystem.directory = "/var/lib/loki/chunks";
+          };
+
+          limits_config = {
+            reject_old_samples = true;
+            reject_old_samples_max_age = "168h";
+            allow_structured_metadata = false;
+          };
+        };
+      };
+    })
+    (mkIf cfg.promtail.enable {
+      services.promtail = {
+        enable = true;
+        configuration = {
+
+          server = {
+            http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+            http_listen_port = 28183;
+            grpc_listen_port = 0;
+          };
+
+          positions.filename = "/tmp/positions.yml";
+
+          clients = [
+            {
+              url = "http://thorite.coho-tet.ts.net:${toString port-loki}/loki/api/v1/push";
+            }
+          ];
+
+          scrape_configs = [
+            {
+              job_name = "journal";
+              # Copied from Mic92's config
+              journal = {
+                max_age = "12h";
+                json = true;
+                labels.job = "systemd-journal";
+              };
+              pipeline_stages = [
+                {
+                  json.expressions = {
+                    transport = "_TRANSPORT";
+                    unit = "_SYSTEMD_UNIT";
+                    msg = "MESSAGE";
+                    coredump_cgroup = "COREDUMP_CGROUP";
+                    coredump_exe = "COREDUMP_EXE";
+                    coredump_cmdline = "COREDUMP_CMDLINE";
+                    coredump_uid = "COREDUMP_UID";
+                    coredump_gid = "COREDUMP_GID";
+                  };
+                }
+                {
+                  # Set the unit (defaulting to the transport like audit and kernel)
+                  template = {
+                    source = "unit";
+                    template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
+                  };
+                }
+                {
+                  regex = {
+                    expression = "(?P<coredump_unit>[^/]+)$";
+                    source = "coredump_cgroup";
+                  };
+                }
+                {
+                  template = {
+                    source = "msg";
+                    # FIXME would be cleaner to have this in a match block, but could not get it to work
+                    template = "{{if .coredump_exe}}{{.coredump_exe}} core dumped (user: {{.coredump_uid}}/{{.coredump_gid}}, command: {{.coredump_cmdline}}){{else}}{{.msg}}{{end}}";
+                  };
+                }
+                { labels.coredump_unit = "coredump_unit"; }
+                {
+                  # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
+                  replace = {
+                    source = "unit";
+                    expression = "^(session-\\d+.scope)$";
+                    replace = "session.scope";
+                  };
+                }
+                { labels.unit = "unit"; }
+                {
+                  # Write the proper message instead of JSON
+                  output.source = "msg";
+                }
+                # silence nscd:
+                # ignore random portscans on the internet
+                { drop.expression = "refused connection: IN="; }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__journal__hostname" ];
+                  target_label = "host";
+                }
+              ];
+            }
+            # {
+            #   job_name = "caddy-access";
+            #   file_sd_configs = {
+            #     files = [
+            #       "/var/log/caddy/*.log"
+            #     ];
+            #     refresh_interval = "5m";
+            #   };
+            # }
+          ];
+        };
+      };
+    })
+  ];
+}