diff --git a/machines/massicot/kanidm-provision.nix b/machines/massicot/kanidm-provision.nix index 9eb10dd..0fdb7b1 100644 --- a/machines/massicot/kanidm-provision.nix +++ b/machines/massicot/kanidm-provision.nix @@ -61,7 +61,6 @@ hedgedoc = { displayName = "HedgeDoc"; originUrl = "https://docs.xinyang.life/"; - originLanding = "https://docs.xinyang.life/auth/oauth2"; allowInsecureClientDisablePkce = true; scopeMaps = { hedgedoc-users = [ "openid" "email" "profile" ]; diff --git a/machines/massicot/services.nix b/machines/massicot/services.nix index f7c9b6b..6c87d4a 100644 --- a/machines/massicot/services.nix +++ b/machines/massicot/services.nix @@ -62,19 +62,6 @@ in group = "kanidm"; }; }; - - services.ntfy-sh = { - enable = true; - group = "caddy"; - settings = { - listen-unix = "/var/run/ntfy-sh/ntfy.sock"; - listen-unix-mode = 432; # octal 0660 - base-url = "https://ntfy.xinyang.life"; - }; - }; - - systemd.services.ntfy-sh.serviceConfig.RuntimeDirectory = "ntfy-sh"; - services.kanidm = { package = pkgs.kanidm.withSecretProvisioning; enableServer = true; @@ -174,11 +161,6 @@ in }; users.groups.git = { }; - users.users = { - ${config.services.caddy.user}.extraGroups = [ - config.services.ntfy-sh.group - ]; - }; services.caddy = { enable = true; @@ -209,14 +191,5 @@ in } } ''; - virtualHosts."https://ntfy.xinyang.life".extraConfig = '' - reverse_proxy unix/${config.services.ntfy-sh.settings.listen-unix} - @httpget { - protocol http - method GET - path_regexp ^/([-_a-z0-9]{0,64}$|docs/|static/) - } - redir @httpget https://{host}{uri} - ''; }; } diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index 0b31ac1..7908b49 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -5,7 +5,7 @@ ./common-settings/nix-conf.nix ./restic.nix ./vaultwarden.nix - ./prometheus + ./prometheus.nix ./hedgedoc.nix ./sing-box.nix ./kanidm-client.nix diff --git a/modules/nixos/prometheus.nix b/modules/nixos/prometheus.nix new file mode 100644 index 0000000..b4a02cc --- /dev/null +++ b/modules/nixos/prometheus.nix @@ -0,0 +1,245 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.custom.prometheus; +in +{ + options = { + custom.prometheus = { + enable = mkEnableOption "Prometheus instance"; + exporters = { + enable = mkOption { + type = types.bool; + default = false; + description = "Enable Prometheus exporter on every supported services"; + }; + }; + grafana = { + enable = mkEnableOption "Grafana Cloud"; + password_file = mkOption { + type = types.path; + }; + }; + }; + }; + + config = mkIf cfg.enable (mkMerge [ + { + services.tailscale = { + enable = true; + permitCertUid = config.services.caddy.user; + }; + + services.caddy = { + enable = true; + virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' + reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} + ''; + }; + + services.caddy.globalConfig = '' + servers { + metrics + } + ''; + services.restic.server.prometheus = cfg.enable; + services.gotosocial.settings = mkIf cfg.enable { + metrics-enabled = true; + }; + services.ntfy-sh.settings.enable-metrics = true; + + services.prometheus = mkIf cfg.enable + { + enable = true; + port = 9091; + globalConfig.external_labels = { hostname = config.networking.hostName; }; + remoteWrite = mkIf cfg.grafana.enable [ + { + name = "grafana"; + url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"; + basic_auth = { + username = "1340065"; + password_file = cfg.grafana.password_file; + }; + } + ]; + exporters = { + node = { + enable = true; + enabledCollectors = [ + "conntrack" + "diskstats" + "entropy" + "filefd" + "filesystem" + "loadavg" + "meminfo" + "netdev" + "netstat" + "stat" + "time" + "vmstat" + "systemd" + "logind" + "interrupts" + "ksmd" + ]; + port = 9100; + }; + }; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } + ]; + } + { + job_name = "node"; + static_configs = [ + { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } + ]; + } + ]; + + alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + logLevel = "debug"; + configuration = { + route = { + receiver = "ntfy"; + }; + receivers = [ + { + name = "ntfy"; + webhook_configs = [ + { + url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' + Alert {{.status}} + {{range .alerts}}-----{{range $k,$v := .labels}} + {{$k}}={{$v}}{{end}} + {{end}} + ''}"; + send_resolved = true; + } + ]; + } + ]; + }; + }; + + alertmanagers = [ + { + scheme = "http"; + static_configs = [ + { + targets = [ + "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" + ]; + } + ]; + } + ]; + + rules = let mkRule = condition: { ... }@rule: (if condition then [ rule ] else [ ]); in [ + (lib.generators.toYAML { } { + groups = (mkRule true + { + name = "system_alerts"; + rules = [ + { + alert = "SystemdFailedUnits"; + expr = "node_systemd_unit_state{state=\"failed\"} > 0"; + for = "5m"; + labels = { severity = "critical"; }; + annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; }; + } + { + alert = "HighLoadAverage"; + expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})"; + for = "1m"; + labels = { severity = "warning"; }; + annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; }; + } + { + alert = "HighTransmitTraffic"; + expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000"; + for = "1m"; + labels = { severity = "warning"; }; + annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; }; + } + ]; + }) ++ (mkRule config.services.restic.server.enable { + name = "restic_alerts"; + rules = [ + { + alert = "ResticCheckFailed"; + expr = "restic_check_success == 0"; + for = "5m"; + labels = { severity = "critical"; }; + annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; }; + } + { + alert = "ResticOutdatedBackup"; + expr = "time() - restic_backup_timestamp > 518400"; + for = "0m"; + labels = { severity = "critical"; }; + annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; }; + } + ]; + }) ++ (mkRule config.services.caddy.enable { + name = "caddy_alerts"; + rules = [ + { + alert = "UpstreamHealthy"; + expr = "caddy_reverse_proxy_upstreams_healthy == 0"; + for = "5m"; + labels = { severity = "critical"; }; + annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; }; + } + { + alert = "HighRequestLatency"; + expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 0.5"; + for = "2m"; + labels = { severity = "warning"; }; + annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; }; + } + ]; + }); + }) + ]; + }; + } + { + services.prometheus.scrapeConfigs = [ + (mkIf config.services.caddy.enable { + job_name = "caddy"; + static_configs = [ + { targets = [ "localhost:2019" ]; } + ]; + }) + (mkIf config.services.restic.server.enable { + job_name = "restic"; + static_configs = [ + { targets = [ config.services.restic.server.listenAddress ]; } + ]; + }) + (mkIf config.services.gotosocial.enable { + job_name = "gotosocial"; + static_configs = [ + { targets = [ "localhost:${toString config.services.gotosocial.settings.port}" ]; } + ]; + }) + (mkIf config.services.ntfy-sh.enable { + job_name = "ntfy-sh"; + static_configs = [ + { targets = [ "auth.xinyang.life" ]; } + ]; + }) + ]; + } + ]); +} diff --git a/modules/nixos/prometheus/caddy.nix b/modules/nixos/prometheus/caddy.nix deleted file mode 100644 index a62b639..0000000 --- a/modules/nixos/prometheus/caddy.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf cfg.enable { - services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable '' - servers { - metrics - } - ''; - - services.prometheus.scrapeConfigs = [ - (lib.mkIf cfg.exporters.caddy.enable { - job_name = "caddy"; - static_configs = [ - { targets = [ "127.0.0.1:2019" ]; } - ]; - }) - ]; - - custom.prometheus.ruleModules = [ - (lib.mkIf cfg.exporters.caddy.enable { - name = "caddy_alerts"; - rules = [ - { - alert = "UpstreamHealthy"; - expr = "caddy_reverse_proxy_upstreams_healthy != 1"; - for = "5m"; - labels = { severity = "critical"; }; - annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; }; - } - { - alert = "HighRequestLatency"; - expr = "histogram_quantile(0.95, rate(caddy_http_request_duration_seconds_bucket[10m])) > 5"; - for = "2m"; - labels = { severity = "warning"; }; - annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; }; - } - ]; - }) - ]; - }; - -} diff --git a/modules/nixos/prometheus/default.nix b/modules/nixos/prometheus/default.nix deleted file mode 100644 index 803b9aa..0000000 --- a/modules/nixos/prometheus/default.nix +++ /dev/null @@ -1,194 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; - -let - cfg = config.custom.prometheus; - exporterCfg = config.custom.prometheus.exporters; - mkExporterOption = enableOption: (mkOption { - type = types.bool; - default = enableOption; - description = "Enable this exporter"; - }); - - mkRulesOption = mkOption { - type = types.listOf (types.submodule { - options = { - name = mkOption { - type = lib.types.str; - }; - rules = mkOption { - type = lib.types.listOf lib.types.attrs; - }; - }; - }); - }; -in -{ - imports = [ - ./caddy.nix - ./gotosocial.nix - ./ntfy-sh.nix - ./restic.nix - ]; - - options = { - custom.prometheus = { - enable = mkEnableOption "Prometheus instance"; - exporters = { - enable = mkOption { - type = types.bool; - default = false; - description = "Enable Prometheus exporter on every supported services"; - }; - - restic.enable = mkExporterOption config.services.restic.server.enable; - caddy.enable = mkExporterOption config.services.caddy.enable; - gotosocial.enable = mkExporterOption config.services.gotosocial.enable; - ntfy-sh.enable = mkExporterOption config.services.gotosocial.enable; - }; - grafana = { - enable = mkEnableOption "Grafana Cloud"; - password_file = mkOption { - type = types.path; - }; - }; - ruleModules = mkRulesOption; - }; - }; - - config = mkIf cfg.enable - { - services.tailscale = { - enable = true; - permitCertUid = config.services.caddy.user; - openFirewall = true; - }; - - services.caddy = { - enable = true; - virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' - reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} - ''; - }; - - services.prometheus = mkIf cfg.enable - { - enable = true; - port = 9091; - globalConfig.external_labels = { hostname = config.networking.hostName; }; - remoteWrite = mkIf cfg.grafana.enable [ - { - name = "grafana"; - url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"; - basic_auth = { - username = "1340065"; - password_file = cfg.grafana.password_file; - }; - } - ]; - exporters = { - node = { - enable = true; - enabledCollectors = [ - "diskstats" - "loadavg" - "time" - "systemd" - ]; - port = 9100; - }; - }; - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ - { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } - ]; - } - { - job_name = "node"; - static_configs = [ - { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } - ]; - } - ]; - - alertmanager = { - enable = true; - listenAddress = "127.0.0.1"; - logLevel = "debug"; - configuration = { - route = { - receiver = "ntfy"; - }; - receivers = [ - { - name = "ntfy"; - webhook_configs = [ - { - url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' - Alert {{.status}} - {{range .alerts}}-----{{range $k,$v := .labels}} - {{$k}}={{$v}}{{end}} - {{end}} - ''}"; - send_resolved = true; - } - ]; - } - ]; - }; - }; - - alertmanagers = [ - { - scheme = "http"; - static_configs = [ - { - targets = [ - "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" - ]; - } - ]; - } - ]; - rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ]; - }; - custom.prometheus.ruleModules = [ - { - name = "system_alerts"; - rules = [ - { - alert = "SystemdFailedUnits"; - expr = "node_systemd_unit_state{state=\"failed\"} > 0"; - for = "5m"; - labels = { severity = "critical"; }; - annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; }; - } - { - alert = "HighLoadAverage"; - expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})"; - for = "1m"; - labels = { severity = "warning"; }; - annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; }; - } - { - alert = "HighTransmitTraffic"; - expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000"; - for = "1m"; - labels = { severity = "warning"; }; - annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; }; - } - { - alert = "NetworkTrafficExceedLimit"; - expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200''; - for = "0m"; - labels = { severity = "critical"; }; - annotations = { summary = "Outbound network traffic exceed 300GB for last 30 day"; }; - } - ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/gotosocial.nix b/modules/nixos/prometheus/gotosocial.nix deleted file mode 100644 index 4870e88..0000000 --- a/modules/nixos/prometheus/gotosocial.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf cfg.exporters.gotosocial.enable { - services.gotosocial.settings = lib.mkIf cfg.exporters.gotosocial.enable { - metrics-enabled = true; - }; - services.prometheus.scrapeConfigs = [ - { - job_name = "gotosocial"; - static_configs = [ - { targets = [ "localhost:8080" ]; } - ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/ntfy-sh.nix b/modules/nixos/prometheus/ntfy-sh.nix deleted file mode 100644 index 35d62ff..0000000 --- a/modules/nixos/prometheus/ntfy-sh.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf cfg.enable { - services.ntfy-sh.settings.enable-metrics = true; - services.prometheus.scrapeConfigs = [ - (lib.mkIf cfg.exporters.ntfy-sh.enable { - job_name = "ntfy-sh"; - static_configs = [ - { targets = [ "ntfy.xinyang.life" ]; } - ]; - }) - ]; - }; -} diff --git a/modules/nixos/prometheus/restic.nix b/modules/nixos/prometheus/restic.nix deleted file mode 100644 index 80f0316..0000000 --- a/modules/nixos/prometheus/restic.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf cfg.enable { - services.restic.server.prometheus = true; - - services.prometheus.scrapeConfigs = [ - (lib.mkIf cfg.exporters.restic.enable { - job_name = "restic"; - static_configs = [ - { targets = [ config.services.restic.server.listenAddress ]; } - ]; - }) - ]; - - custom.prometheus.ruleModules = [ - (lib.mkIf cfg.exporters.restic.enable { - name = "restic_alerts"; - rules = [ - { - alert = "ResticCheckFailed"; - expr = "restic_check_success == 0"; - for = "5m"; - labels = { severity = "critical"; }; - annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; }; - } - { - alert = "ResticOutdatedBackup"; - expr = "time() - restic_backup_timestamp > 518400"; - for = "0m"; - labels = { severity = "critical"; }; - annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; }; - } - ]; - }) - ]; - }; - -}