modules/monitoring: add loki and promtail

modules/nixos/common-settings/mainland.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    mkDefault
+    ;
+
+  cfg = config.inMainland;
+in
+{
+  options.inMainland = mkOption {
+    type = types.bool;
+    default = config.time.timeZone == "Asia/Shanghai";
+  };
+
+  config = mkIf cfg {
+    nix.settings.extra-substituters = [
+      "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
+    ];
+
+    networking.timeServers = [
+      "cn.ntp.org.cn"
+      "ntp.ntsc.ac.cn"
+    ];
+
+    services.dae = {
+      enable = mkDefault true;
+    };
+  };
+}

modules/nixos/common-settings/nix-conf.nix

@@ -21,7 +21,6 @@ in
       default = true;
       type = types.bool;
     };
-    enableMirrors = mkEnableOption "cache.nixos.org mirrors in Mainland China";
     signing = {
       enable = mkEnableOption "Sign locally-built paths";
       keyFile = mkOption {
@@ -55,10 +54,6 @@ in
         "https://cache.garnix.io"
       ];
 
-      extra-substituters = mkIf cfg.enableMirrors [
-        "https://mirrors.cernet.edu.cn/nix-channels/store?priority=20"
-      ];
-
       trusted-public-keys = [
         "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
         "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g="

modules/nixos/common-settings/proxy-server.nix

@@ -9,8 +9,6 @@ let
     mkIf
     mkEnableOption
     mkOption
-    mkDefault
-    types
     ;
 
   cfg = config.commonSettings.proxyServer;
@@ -26,6 +24,9 @@ let
   mkSingConfig =
     { uuid, password, ... }:
     {
+      log = {
+        level = "warn";
+      };
       inbounds =
         [
           {

modules/nixos/default.nix

@@ -4,6 +4,7 @@
     ./common-settings/autoupgrade.nix
     ./common-settings/nix-conf.nix
     ./common-settings/proxy-server.nix
+    ./common-settings/mainland.nix
     ./disk-partitions
     ./restic.nix
     ./vaultwarden.nix

modules/nixos/monitor/default.nix

@@ -28,6 +28,7 @@ in
   imports = [
     ./exporters.nix
     ./grafana.nix
+    ./loki.nix
   ];
 
   options = {

modules/nixos/monitor/loki.nix

@@ -0,0 +1,166 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkMerge
+    ;
+  cfg = config.custom.monitoring;
+  port-loki = 3100;
+in
+{
+  options = {
+    custom.monitoring = {
+      loki.enable = mkEnableOption "loki";
+      promtail.enable = mkEnableOption "promtail";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.loki.enable {
+      services.loki = {
+        enable = true;
+        configuration = {
+          auth_enabled = false;
+          server.http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+          server.http_listen_port = port-loki;
+
+          common = {
+            ring = {
+              instance_addr = "${config.networking.hostName}.coho-tet.ts.net";
+              kvstore.store = "inmemory";
+            };
+            replication_factor = 1;
+            path_prefix = "/var/lib/loki";
+          };
+
+          schema_config.configs = [
+            {
+              from = "2024-12-01";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+
+          storage_config = {
+            filesystem.directory = "/var/lib/loki/chunks";
+          };
+
+          limits_config = {
+            reject_old_samples = true;
+            reject_old_samples_max_age = "168h";
+            allow_structured_metadata = false;
+          };
+        };
+      };
+    })
+    (mkIf cfg.promtail.enable {
+      services.promtail = {
+        enable = true;
+        configuration = {
+
+          server = {
+            http_listen_address = "${config.networking.hostName}.coho-tet.ts.net";
+            http_listen_port = 28183;
+            grpc_listen_port = 0;
+          };
+
+          positions.filename = "/tmp/positions.yml";
+
+          clients = [
+            {
+              url = "http://thorite.coho-tet.ts.net:${toString port-loki}/loki/api/v1/push";
+            }
+          ];
+
+          scrape_configs = [
+            {
+              job_name = "journal";
+              # Copied from Mic92's config
+              journal = {
+                max_age = "12h";
+                json = true;
+                labels.job = "systemd-journal";
+              };
+              pipeline_stages = [
+                {
+                  json.expressions = {
+                    transport = "_TRANSPORT";
+                    unit = "_SYSTEMD_UNIT";
+                    msg = "MESSAGE";
+                    coredump_cgroup = "COREDUMP_CGROUP";
+                    coredump_exe = "COREDUMP_EXE";
+                    coredump_cmdline = "COREDUMP_CMDLINE";
+                    coredump_uid = "COREDUMP_UID";
+                    coredump_gid = "COREDUMP_GID";
+                  };
+                }
+                {
+                  # Set the unit (defaulting to the transport like audit and kernel)
+                  template = {
+                    source = "unit";
+                    template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
+                  };
+                }
+                {
+                  regex = {
+                    expression = "(?P<coredump_unit>[^/]+)$";
+                    source = "coredump_cgroup";
+                  };
+                }
+                {
+                  template = {
+                    source = "msg";
+                    # FIXME would be cleaner to have this in a match block, but could not get it to work
+                    template = "{{if .coredump_exe}}{{.coredump_exe}} core dumped (user: {{.coredump_uid}}/{{.coredump_gid}}, command: {{.coredump_cmdline}}){{else}}{{.msg}}{{end}}";
+                  };
+                }
+                { labels.coredump_unit = "coredump_unit"; }
+                {
+                  # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
+                  replace = {
+                    source = "unit";
+                    expression = "^(session-\\d+.scope)$";
+                    replace = "session.scope";
+                  };
+                }
+                { labels.unit = "unit"; }
+                {
+                  # Write the proper message instead of JSON
+                  output.source = "msg";
+                }
+                # silence nscd:
+                # ignore random portscans on the internet
+                { drop.expression = "refused connection: IN="; }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__journal__hostname" ];
+                  target_label = "host";
+                }
+              ];
+            }
+            # {
+            #   job_name = "caddy-access";
+            #   file_sd_configs = {
+            #     files = [
+            #       "/var/log/caddy/*.log"
+            #     ];
+            #     refresh_interval = "5m";
+            #   };
+            # }
+          ];
+        };
+      };
+    })
+  ];
+}