xin/nixos-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

modules/prometheus: add basic auth

Browse source
This commit is contained in:
xinyangli 2024-11-30 17:35:46 +08:00
parent 97fcdefc2b
commit 74b67e1854
Signed by: xin
SSH key fingerprint: SHA256:UU5pRTl7NiLFJbWJZa+snLylZSXIz5rgHmwjzv8v4oE
10 changed files with 184 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/nixos/default.nix
View file

@ -7,7 +7,7 @@
./disk-partitions
./restic.nix
./vaultwarden.nix
./prometheus
./monitor
./hedgedoc.nix
./sing-box.nix
./kanidm-client.nix

177
modules/nixos/monitor/default.nix
View file

@ -8,6 +8,7 @@ let
mkEnableOption
mkOption
mkIf
mkMerge
types
;
cfg = config.custom.prometheus;
@ -58,91 +59,113 @@ in
};
};
config = mkIf cfg.enable {
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
openFirewall = true;
};
config = mkMerge [
{
sops.secrets = {
"prometheus/metrics_username" = {
sopsFile = ../../../machines/secrets.yaml;
group = "prometheus-auth";
mode = "0440";
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.prometheus = mkIf cfg.enable {
enable = true;
port = 9091;
globalConfig.external_labels = {
hostname = config.networking.hostName;
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
"prometheus/metrics_password" = {
sopsFile = ../../../machines/secrets.yaml;
group = "prometheus-auth";
mode = "0440";
};
};
alertmanagers = [
users.groups.prometheus-auth.members = [
"prometheus"
];
}
(mkIf cfg.enable {
services.tailscale = {
enable = true;
permitCertUid = config.services.caddy.user;
openFirewall = true;
};
services.caddy = {
enable = true;
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
'';
};
services.prometheus = mkIf cfg.enable {
enable = true;
port = 9091;
globalConfig.external_labels = {
hostname = config.networking.hostName;
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
logLevel = "debug";
configuration = {
route = {
receiver = "ntfy";
};
receivers = [
{
name = "ntfy";
webhook_configs = [
{
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
Alert {{.status}}
{{range .alerts}}-----{{range $k,$v := .labels}}
{{$k}}={{$v}}{{end}}
{{end}}
''}";
send_resolved = true;
}
];
}
];
};
};
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
};
custom.prometheus.ruleModules = [
{
scheme = "http";
static_configs = [
name = "prometheus_alerts";
rules = [
{
targets = [
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
];
alert = "JobDown";
expr = "up == 0";
for = "1m";
labels = {
severity = "critical";
};
annotations = {
summary = "Job {{ $labels.job }} down for 1m.";
};
}
];
}
];
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
};
custom.prometheus.ruleModules = [
{
name = "prometheus_alerts";
rules = [
{
alert = "JobDown";
expr = "up == 0";
for = "1m";
labels = {
severity = "critical";
};
annotations = {
summary = "Job {{ $labels.job }} down for 1m.";
};
}
];
}
];
};
})
];
}

34
modules/nixos/monitor/exporters.nix
View file

@ -46,6 +46,21 @@ in
);
};
# gotosocial
sops.templates."gotosocial_metrics.env" = {
content = ''
GTS_METRICS_AUTH_ENABLED=true
GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
'';
group = "prometheus-auth";
mode = "0440";
};
systemd.services.gotosocial.serviceConfig = {
EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ];
SupplementaryGroups = [ "prometheus-auth" ];
};
services.gotosocial.settings = {
metrics-enabled = true;
};
@ -55,7 +70,24 @@ in
};
services.restic.server.prometheus = true;
systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
# miniflux
sops.templates."miniflux_metrics_env" = {
content = ''
METRICS_COLLECTOR=1
LOG_LEVEL=debug
METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
'';
group = "prometheus-auth";
mode = "0440";
};
systemd.services.miniflux.serviceConfig = {
EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ];
SupplementaryGroups = [ "prometheus-auth" ];
};
services.ntfy-sh.settings.enable-metrics = true;
services.caddy.globalConfig = ''