xin/nixos-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

modules/prometheus: move to monitor directory

Browse source
This commit is contained in:
xinyangli 2024-11-30 12:56:54 +08:00
parent 9e3af9a535
commit 97fcdefc2b
Signed by: xin
SSH key fingerprint: SHA256:UU5pRTl7NiLFJbWJZa+snLylZSXIz5rgHmwjzv8v4oE
4 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

148
modules/nixos/monitor/default.nix Normal file
View file

@ -0,0 +1,148 @@
{
config,
lib,
...
}:
let
inherit (lib)
mkEnableOption
mkOption
mkIf
types
;
cfg = config.custom.prometheus;
mkRulesOption = mkOption {
type = types.listOf (
types.submodule {
options = {
name = mkOption { type = lib.types.str; };
rules = mkOption { type = lib.types.listOf lib.types.attrs; };
};
}
);
};
in
{
imports = [
./exporters.nix
./grafana.nix
];
options = {
custom.monitoring = {
grafana = {
enable = mkEnableOption "grafana with oauth only";
};
};
custom.prometheus = {
enable = mkEnableOption "Prometheus instance";
ruleModules = mkRulesOption;
exporters = {
enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
node = {
enable = mkEnableOption "node exporter";
listenAddress = mkOption {
type = types.str;
default = "${config.networking.hostName}.coho-tet.ts.net";
};
};
blackbox = {
enable = mkEnableOption "blackbox exporter";
listenAddress = mkOption {
type = types.str;
default = "${config.networking.hostName}.coho-tet.ts.net";
};
};
};
};
};
config = mkIf cfg.enable {
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
openFirewall = true;
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.prometheus = mkIf cfg.enable {
enable = true;
port = 9091;
globalConfig.external_labels = {
hostname = config.networking.hostName;
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
};
custom.prometheus.ruleModules = [
{
name = "prometheus_alerts";
rules = [
{
alert = "JobDown";
expr = "up == 0";
for = "1m";
labels = {
severity = "critical";
};
annotations = {
summary = "Job {{ $labels.job }} down for 1m.";
};
}
];
}
];
};
}

70
modules/nixos/monitor/exporters.nix Normal file
View file

@ -0,0 +1,70 @@
{
config,
pkgs,
lib,
...
}:
let
inherit (lib) mkIf;
cfg = config.custom.prometheus.exporters;
in
{
config = {
systemd.services.tailscaled.after =
(lib.optional cfg.node.enable "prometheus-node-exporters.service")
++ (lib.optional cfg.blackbox.enable "prometheus-blackbox-exporters.service")
++ (lib.optional config.services.caddy.enable "caddy.service");
services.prometheus.exporters.node = mkIf cfg.node.enable {
enable = true;
enabledCollectors = [
"loadavg"
"time"
"systemd"
];
listenAddress = cfg.node.listenAddress;
port = 9100;
};
services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
enable = true;
listenAddress = cfg.blackbox.listenAddress;
configFile = pkgs.writeText "blackbox.config.yaml" (
lib.generators.toYAML { } {
modules = {
tcp4_connect = {
prober = "tcp";
tcp = {
ip_protocol_fallback = false;
preferred_ip_protocol = "ip4";
tls = false;
};
timeout = "15s";
};
};
}
);
};
services.gotosocial.settings = {
metrics-enabled = true;
};
services.immich.environment = {
IMMICH_TELEMETRY_INCLUDE = "all";
};
services.restic.server.prometheus = true;
systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
services.ntfy-sh.settings.enable-metrics = true;
services.caddy.globalConfig = ''
servers {
metrics
}
admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
}
'';
};
}

43
modules/nixos/monitor/grafana.nix Normal file
View file

@ -0,0 +1,43 @@
{ config, lib, ... }:
let
cfg = config.custom.monitoring.grafana;
in
{
config = lib.mkIf cfg.enable {
sops.templates."grafana.env".content = ''
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${config.sops.placeholder."grafana/oauth_secret"}
'';
services.grafana = {
enable = true;
settings = {
server = {
http_addr = "127.0.0.1";
http_port = 3003;
root_url = "https://grafana.xinyang.life";
domain = "grafana.xinyang.life";
};
"auth.generic_oauth" = {
enabled = true;
name = "Kanidm";
client_id = "grafana";
scopes = "openid,profile,email,groups";
auth_url = "https://auth.xinyang.life/ui/oauth2";
token_url = "https://auth.xinyang.life/oauth2/token";
api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
use_pkce = true;
use_refresh_token = true;
allow_sign_up = true;
login_attribute_path = "preferred_username";
groups_attribute_path = "groups";
role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
allow_assign_grafana_admin = true;
auto_login = true;
};
"auth" = {
disable_login_form = true;
};
};
};
systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.templates."grafana.env".path;
};
}

58
modules/nixos/monitor/restic.nix Normal file
View file

@ -0,0 +1,58 @@
{ config, lib, ... }:
let
cfg = config.custom.prometheus;
in
{
config = {
services.restic.server.prometheus = true;
custom.prometheus.templates.scrape.mkResticScrapes =
{
address,
port ? null,
...
}:
let
portStr = if port then ":${toString port}" else "";
in
[
(lib.mkIf cfg.exporters.restic.enable {
job_name = "restic";
static_configs = [ { targets = [ "${address}${portStr}" ]; } ];
})
];
custom.prometheus.templates.rules.mkResticRules = [
{
name = "restic_alerts";
rules = [
{
alert = "ResticCheckFailed";
expr = "restic_check_success == 0";
for = "5m";
labels = {
severity = "critical";
};
annotations = {
summary = "Restic check failed (instance {{ $labels.instance }})";
description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}";
};
}
{
alert = "ResticOutdatedBackup";
expr = "time() - restic_backup_timestamp > 518400";
for = "0m";
labels = {
severity = "critical";
};
annotations = {
summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated";
description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}";
};
}
];
}
];
};
}