treewide: apply the new rfc nixfmt
This commit is contained in:
parent
e702d503b9
commit
7a795e5ed9
SSH key fingerprint:
SHA256:qZ/tzd8lYRtUFSrfBDBMcUqV4GHKxqeqRA3huItgvbk
53 changed files with 1371 additions and 888 deletions
|
|
@ -1,4 +1,9 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.custom.prometheus;
|
||||
in
|
||||
|
|
@ -8,7 +13,7 @@ in
|
|||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
configFile = pkgs.writeText "blackbox.config.yaml" (
|
||||
lib.generators.toYAML {} {
|
||||
lib.generators.toYAML { } {
|
||||
modules = {
|
||||
tcp4_connect = {
|
||||
prober = "tcp";
|
||||
|
|
@ -25,7 +30,7 @@ in
|
|||
};
|
||||
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
{
|
||||
job_name = "blackbox";
|
||||
scrape_interval = "1m";
|
||||
metrics_path = "/probe";
|
||||
|
|
@ -73,8 +78,13 @@ in
|
|||
alert = "HighProbeLatency";
|
||||
expr = "probe_duration_seconds > 0.5";
|
||||
for = "2m";
|
||||
labels = { severity = "warning"; };
|
||||
annotations = { summary = "High request latency on {{ $labels.instance }}"; description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; };
|
||||
labels = {
|
||||
severity = "warning";
|
||||
};
|
||||
annotations = {
|
||||
summary = "High request latency on {{ $labels.instance }}";
|
||||
description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes.";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,9 +13,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "caddy";
|
||||
static_configs = [
|
||||
{ targets = [ "127.0.0.1:2019" ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ];
|
||||
}
|
||||
];
|
||||
|
||||
|
|
@ -27,8 +25,12 @@ in
|
|||
alert = "UpstreamHealthy";
|
||||
expr = "caddy_reverse_proxy_upstreams_healthy != 1";
|
||||
for = "5m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Upstream {{ $labels.unstream }} not healthy"; };
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Upstream {{ $labels.unstream }} not healthy";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,31 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.custom.prometheus;
|
||||
mkExporterOption = enableOption: (mkOption {
|
||||
type = types.bool;
|
||||
default = enableOption;
|
||||
description = "Enable this exporter";
|
||||
});
|
||||
mkExporterOption =
|
||||
enableOption:
|
||||
(mkOption {
|
||||
type = types.bool;
|
||||
default = enableOption;
|
||||
description = "Enable this exporter";
|
||||
});
|
||||
|
||||
mkRulesOption = mkOption {
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
name = mkOption {
|
||||
type = lib.types.str;
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
name = mkOption { type = lib.types.str; };
|
||||
rules = mkOption { type = lib.types.listOf lib.types.attrs; };
|
||||
};
|
||||
rules = mkOption {
|
||||
type = lib.types.listOf lib.types.attrs;
|
||||
};
|
||||
};
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
in
|
||||
{
|
||||
|
|
@ -54,153 +59,172 @@ in
|
|||
};
|
||||
grafana = {
|
||||
enable = mkEnableOption "Grafana Cloud";
|
||||
password_file = mkOption {
|
||||
type = types.path;
|
||||
};
|
||||
password_file = mkOption { type = types.path; };
|
||||
};
|
||||
ruleModules = mkRulesOption;
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable
|
||||
{
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
permitCertUid = config.services.caddy.user;
|
||||
openFirewall = true;
|
||||
};
|
||||
config = mkIf cfg.enable {
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
permitCertUid = config.services.caddy.user;
|
||||
openFirewall = true;
|
||||
};
|
||||
|
||||
services.caddy = {
|
||||
enable = true;
|
||||
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
||||
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
||||
'';
|
||||
};
|
||||
services.caddy = {
|
||||
enable = true;
|
||||
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
||||
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
||||
'';
|
||||
};
|
||||
|
||||
services.prometheus = mkIf cfg.enable
|
||||
services.prometheus = mkIf cfg.enable {
|
||||
enable = true;
|
||||
port = 9091;
|
||||
globalConfig.external_labels = {
|
||||
hostname = config.networking.hostName;
|
||||
};
|
||||
remoteWrite = mkIf cfg.grafana.enable [
|
||||
{
|
||||
name = "grafana";
|
||||
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
|
||||
basic_auth = {
|
||||
username = "1340065";
|
||||
password_file = cfg.grafana.password_file;
|
||||
};
|
||||
}
|
||||
];
|
||||
exporters = {
|
||||
node = {
|
||||
enable = true;
|
||||
port = 9091;
|
||||
globalConfig.external_labels = { hostname = config.networking.hostName; };
|
||||
remoteWrite = mkIf cfg.grafana.enable [
|
||||
{
|
||||
name = "grafana";
|
||||
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
|
||||
basic_auth = {
|
||||
username = "1340065";
|
||||
password_file = cfg.grafana.password_file;
|
||||
};
|
||||
}
|
||||
enabledCollectors = [
|
||||
"loadavg"
|
||||
"time"
|
||||
"systemd"
|
||||
];
|
||||
exporters = {
|
||||
node = {
|
||||
enable = true;
|
||||
enabledCollectors = [
|
||||
"loadavg"
|
||||
"time"
|
||||
"systemd"
|
||||
];
|
||||
listenAddress = "127.0.0.1";
|
||||
port = 9100;
|
||||
};
|
||||
};
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "prometheus";
|
||||
static_configs = [
|
||||
{ targets = [ "localhost:${toString config.services.prometheus.port}" ]; }
|
||||
];
|
||||
}
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [
|
||||
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
alertmanager = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
logLevel = "debug";
|
||||
configuration = {
|
||||
route = {
|
||||
receiver = "ntfy";
|
||||
};
|
||||
receivers = [
|
||||
{
|
||||
name = "ntfy";
|
||||
webhook_configs = [
|
||||
{
|
||||
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
|
||||
Alert {{.status}}
|
||||
{{range .alerts}}-----{{range $k,$v := .labels}}
|
||||
{{$k}}={{$v}}{{end}}
|
||||
{{end}}
|
||||
''}";
|
||||
send_resolved = true;
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
alertmanagers = [
|
||||
{
|
||||
scheme = "http";
|
||||
static_configs = [
|
||||
{
|
||||
targets = [
|
||||
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
|
||||
listenAddress = "127.0.0.1";
|
||||
port = 9100;
|
||||
};
|
||||
custom.prometheus.ruleModules = [
|
||||
};
|
||||
scrapeConfigs = [
|
||||
{
|
||||
name = "system_alerts";
|
||||
rules = [
|
||||
job_name = "prometheus";
|
||||
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
|
||||
}
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [
|
||||
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
alertmanager = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
logLevel = "debug";
|
||||
configuration = {
|
||||
route = {
|
||||
receiver = "ntfy";
|
||||
};
|
||||
receivers = [
|
||||
{
|
||||
alert = "SystemdFailedUnits";
|
||||
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
|
||||
for = "5m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Systemd has failed units on {{ $labels.instance }}"; description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; };
|
||||
name = "ntfy";
|
||||
webhook_configs = [
|
||||
{
|
||||
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
|
||||
Alert {{.status}}
|
||||
{{range .alerts}}-----{{range $k,$v := .labels}}
|
||||
{{$k}}={{$v}}{{end}}
|
||||
{{end}}
|
||||
''}";
|
||||
send_resolved = true;
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
alertmanagers = [
|
||||
{
|
||||
scheme = "http";
|
||||
static_configs = [
|
||||
{
|
||||
alert = "HighLoadAverage";
|
||||
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
|
||||
for = "1m";
|
||||
labels = { severity = "warning"; };
|
||||
annotations = { summary = "High load average detected on {{ $labels.instance }}"; description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; };
|
||||
}
|
||||
{
|
||||
alert = "HighTransmitTraffic";
|
||||
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
|
||||
for = "1m";
|
||||
labels = { severity = "warning"; };
|
||||
annotations = { summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; };
|
||||
}
|
||||
{
|
||||
alert = "NetworkTrafficExceedLimit";
|
||||
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
|
||||
for = "0m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Outbound network traffic exceed 300GB for last 30 day"; };
|
||||
}
|
||||
{
|
||||
alert = "JobDown";
|
||||
expr = "up == 0";
|
||||
for = "1m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Job {{ $labels.job }} down for 1m."; };
|
||||
targets = [
|
||||
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
|
||||
};
|
||||
custom.prometheus.ruleModules = [
|
||||
{
|
||||
name = "system_alerts";
|
||||
rules = [
|
||||
{
|
||||
alert = "SystemdFailedUnits";
|
||||
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
|
||||
for = "5m";
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Systemd has failed units on {{ $labels.instance }}";
|
||||
description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
|
||||
};
|
||||
}
|
||||
{
|
||||
alert = "HighLoadAverage";
|
||||
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
|
||||
for = "1m";
|
||||
labels = {
|
||||
severity = "warning";
|
||||
};
|
||||
annotations = {
|
||||
summary = "High load average detected on {{ $labels.instance }}";
|
||||
description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
|
||||
};
|
||||
}
|
||||
{
|
||||
alert = "HighTransmitTraffic";
|
||||
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
|
||||
for = "1m";
|
||||
labels = {
|
||||
severity = "warning";
|
||||
};
|
||||
annotations = {
|
||||
summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
|
||||
description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
|
||||
};
|
||||
}
|
||||
{
|
||||
alert = "NetworkTrafficExceedLimit";
|
||||
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
|
||||
for = "0m";
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Outbound network traffic exceed 300GB for last 30 day";
|
||||
};
|
||||
}
|
||||
{
|
||||
alert = "JobDown";
|
||||
expr = "up == 0";
|
||||
for = "1m";
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Job {{ $labels.job }} down for 1m.";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,9 +10,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "gotosocial";
|
||||
static_configs = [
|
||||
{ targets = [ "localhost:8080" ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ "localhost:8080" ]; } ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ let
|
|||
cfg = config.custom.prometheus;
|
||||
immichEnv = config.services.immich.environment;
|
||||
metricPort =
|
||||
if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv
|
||||
then immichEnv.IMMICH_API_METRICS_PORT
|
||||
else 8081;
|
||||
if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv then
|
||||
immichEnv.IMMICH_API_METRICS_PORT
|
||||
else
|
||||
8081;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) {
|
||||
|
|
@ -16,9 +17,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "immich";
|
||||
static_configs = [
|
||||
{ targets = [ "127.0.0.1:${toString metricPort}" ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ "127.0.0.1:${toString metricPort}" ]; } ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "miniflux";
|
||||
static_configs = [
|
||||
{ targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; } ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "ntfy-sh";
|
||||
static_configs = [
|
||||
{ targets = [ "ntfy.xinyang.life" ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ "ntfy.xinyang.life" ]; } ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@ in
|
|||
services.prometheus.scrapeConfigs = [
|
||||
(lib.mkIf cfg.exporters.restic.enable {
|
||||
job_name = "restic";
|
||||
static_configs = [
|
||||
{ targets = [ config.services.restic.server.listenAddress ]; }
|
||||
];
|
||||
static_configs = [ { targets = [ config.services.restic.server.listenAddress ]; } ];
|
||||
})
|
||||
];
|
||||
|
||||
|
|
@ -23,15 +21,25 @@ in
|
|||
alert = "ResticCheckFailed";
|
||||
expr = "restic_check_success == 0";
|
||||
for = "5m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Restic check failed (instance {{ $labels.instance }})"; description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Restic check failed (instance {{ $labels.instance }})";
|
||||
description = "Restic check failed\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}";
|
||||
};
|
||||
}
|
||||
{
|
||||
alert = "ResticOutdatedBackup";
|
||||
expr = "time() - restic_backup_timestamp > 518400";
|
||||
for = "0m";
|
||||
labels = { severity = "critical"; };
|
||||
annotations = { summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated"; description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}"; };
|
||||
labels = {
|
||||
severity = "critical";
|
||||
};
|
||||
annotations = {
|
||||
summary = "Restic {{ $labels.client_hostname }} / {{ $labels.client_username }} backup is outdated";
|
||||
description = "Restic backup is outdated\\n VALUE = {{ $value }}\\n LABELS = {{ $labels }}";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue