From 74b67e18544a77bc68ede7ea8e003da00c700608 Mon Sep 17 00:00:00 2001 From: xinyangli Date: Sat, 30 Nov 2024 17:35:46 +0800 Subject: [PATCH] modules/prometheus: add basic auth --- .sops.yaml | 16 ++- machines/netdrives.nix | 22 ---- machines/secrets.yaml | 96 ++++----------- machines/sops.nix | 15 +-- machines/thorite/default.nix | 1 - machines/thorite/monitoring.nix | 5 + modules/nixos/default.nix | 2 +- modules/nixos/monitor/default.nix | 177 ++++++++++++++++------------ modules/nixos/monitor/exporters.nix | 34 +++++- overlays/my-lib/prometheus.nix | 10 ++ 10 files changed, 184 insertions(+), 194 deletions(-) delete mode 100644 machines/netdrives.nix diff --git a/.sops.yaml b/.sops.yaml index 8e9c1d8..adfc3d5 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -10,6 +10,13 @@ keys: - &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv - &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96 creation_rules: + - path_regex: machines/secrets.yaml + key_groups: + - age: + - *xin + - *host-calcite + - *host-massicot + - *host-thorite - path_regex: machines/calcite/secrets.yaml key_groups: - age: @@ -58,15 +65,6 @@ creation_rules: - age: - *xin - *host-weilite - - path_regex: machines/secrets.yaml - key_groups: - - age: - - *xin - - *host-calcite - - *host-raspite - - *host-la-00 - - *host-hk-00 - - *host-massicot - path_regex: home/xin/secrets.yaml key_groups: - age: diff --git a/machines/netdrives.nix b/machines/netdrives.nix deleted file mode 100644 index 2fedf53..0000000 --- a/machines/netdrives.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs, config, ... }: -{ - sops.secrets = { - autofs-nas = { - owner = "davfs2"; - }; - autofs-nas-secret = { - path = "/etc/davfs2/secrets"; - }; - }; - fileSystems."/media/nas" = { - device = "https://home.xinyang.life:5244/dav"; - fsType = "davfs"; - options = [ - "uid=1000" - "gid=1000" - "rw" - "_netdev" - ]; - - }; -} diff --git a/machines/secrets.yaml b/machines/secrets.yaml index cedd676..25aa038 100644 --- a/machines/secrets.yaml +++ b/machines/secrets.yaml @@ -1,14 +1,6 @@ -clash_subscription_link: ENC[AES256_GCM,data:uDaX2BE/qRdfXVtckX0VKpu0LN3j0YaxVIPbQt3tGAfdfqFqlp0IzFgNiZBIEcIltYkeEyqFSA0QnttoMb0QYe9f2rtgjztwk10SOGViGaeFWPfkdlHP04qhm5OOOddi3OwT5rUNwvBU79AdCnLJ9QwqMbOaNm/JTtbkcjf8huxc2UcYAQcY/YNJ7aTEhWIw98Ab85aih+w=,iv:pZ189IPPCBjscXzEdgQCRdFlls3TniwDfNCd+H1FFaQ=,tag:dMmGZvppWtkc82b5dTnJwg==,type:str] -autofs-nas: ENC[AES256_GCM,data:LnCKGKARx6Vd99VwAX/6PXOJwo+a7GP8fNmM9yuuC2xITGxtWCsDdOZL1+IA5LS/gbOYINgQWDzWirJF3LCP27BQeLwXYpD7/UAwwVI=,iv:QJzsS5a6vWeoBxkB13yXdVbyn0tt2QTvqj0LaHn6S2g=,tag:D/JKXQIw1EzIh3wjGhHgHg==,type:str] -autofs-nas-secret: ENC[AES256_GCM,data:gbOizRZAvh79HlJWIWeKTk79Ux311XGL1eIswc0P2U2huCibD/ji3kOlSjZXENG+fJQKNz2AlDTk3g2cQQ==,iv:UCaGeE8j4RqJzA0xhu3oB2xvzombzQD3fjLKCWd5fDg=,tag:II4eEMr7f2TDUl1qUcDYXA==,type:str] -github_public_token: ENC[AES256_GCM,data:6Gt+oJcCRHeoLK7CRndMMbszTXSEbnN0nQzsVOnl/+zB4hxbEPD5k/vkkl+cZ/qmxdxFXV0OOsYvktn44Yv1DMUE3mkB0hcAdoyPwLuYM7W3RpOoW3OktH8DRCUi6msvFp3ykpdmIl9WyjVhc/lMwTaYJQyRh1ue,iv:PJSFtJBelyc3rzd6hqjMp+ciU2Q3FTOEXsiq5F2KKTY=,tag:Y/stRg6kwyjjIFZCXS/peg==,type:str] -singbox_sg_server: ENC[AES256_GCM,data:SF2ja6W4TwThwoug5x2KTA==,iv:Vx9wNTdVHkReux4YeQY+0VkC1Wqg/CRkY7frVY/3e50=,tag:7XA9KSoR0GA6FoYRhCv4BQ==,type:str] -singbox_jp_server: ENC[AES256_GCM,data:S3Bs5yVMzyz6vD51GYElOM5h,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:o9d55cZuWmX4NDYexWjvYQ==,type:str] -sing-box: - password: ENC[AES256_GCM,data:xyqmoJEDI5959zHPTVelln/iThtoeDwS,iv:rLyqJsE/4JDf08RlMLLPh+MKJkba9bL0z8jx6bTEfgc=,tag:cgLHdeLIyPvLhRNaVcQ0TQ==,type:str] - uuid: ENC[AES256_GCM,data:lWBCM5wyz6BcUUHdvynkn5y166Kk15jO0EhWUDuhXXhrve5l,iv:RmDJYFnYqIEIShLn25sf4h8AO2E3+3Xa2U9Mff+Xk2w=,tag:SN0DUdwZXKO/VEnozrr5mA==,type:str] -grafana_cloud_api: ENC[AES256_GCM,data:eEvPAwtThK1FMhbrnmSo89+GlWZAF+LQRMLXA2C6f1vR7ZPlXJZGWzjYwDcPlnpiC737/cG14M4kZqvPGBuNub5A83rBS/+FeebvGDIF59L5PC1Ys1jWBB9YRI/L9EU0tvwTTUCvLRA9j28n7Jw7wR6mWXm63XA+OMu8/UbTwbeV/WUQn8vnwqadSUdCnNKJXMsAY+q9t/st0DPm5+aNxA==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:87C+0FVvzDIowE0+QpY1zA==,type:str] -private_dns_address: ENC[AES256_GCM,data:YJxNOH4hsZHResvANEqJRTANhnL4PLp/Pmi/PhgtSTbTKiJKPqudhTEkNg==,iv:8+qG5rQXAKfrykEjt9qrbtyNaBuKvi7EaIWouRqEipY=,tag:VH0w5ZbXcWFGZ9GLavm7/w==,type:str] +prometheus: + metrics_username: ENC[AES256_GCM,data:/CQfOA==,iv:BjhB+uLfjmYHdgpc/+tDJXJ8C1EK9kngQWbo4NleOmE=,tag:JCdqyqGLRh09T25vmufiZw==,type:str] + metrics_password: ENC[AES256_GCM,data:q/xMPuNtlcUFewMdVu6w2Q==,iv:xLohdb5tdxevYFckZoacjSJp2rZ53QKLxK6u3mc3mDw=,tag:B4LrObH1DsnnD5CcuOPOyg==,type:str] sops: kms: [] gcp_kms: [] @@ -18,77 +10,41 @@ sops: - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiRzZVNGFocUN2VzZLTmJz - WlJnUmxhZS92citDRkVZVnJZQU9YWVZORlNjCkgzeWl5dTl1YmpjZGt2anF3dGgr - K1hOSTRmakNrZ2JoNit3NDIzK1FCcWMKLS0tIEdqY3VvR3gxd1JoQlhPR3JvcXBF - K2g4VFpqUEF5RTQ3cmpUSG0xajN2bUUKMuwx5cO1nHokV1NOloXfl9wTBN/+/Rlq - UJKP/qaI23tpyMXN1U40iF20ecO1U5Ad8wAQ61C/tldSVULizDihpw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvRTNjSmw3WkgxMGtVSW9C + ZzFDdWUwY3FLOEZHUmtGdWkwYkd5NXI4S3dvCk1WdUx6Qk1sbzkxQU5TQU53c2lx + bWtNZ0U3cGVnWWd3VGczNmhuVEFTMDgKLS0tIDU4T0EvZzF6d0dJaWoyN0dqOVJl + RDRRS0RYNnI0OEtXNTFrL1R1aVczd28KqVk9onzsphU0pHwqhjpKVQ8hOjdcIRJ0 + 3dsI05nKRGjx/1yZBgGNbOR7LE+w63zR7KBoHYa49FEpWGiy54j2ZQ== -----END AGE ENCRYPTED FILE----- - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYXlwdytVSm1SQzRubHdX - dHhrTWxyamo3OFRraEVRQ3plK1cwUWt0a1JRCkdqaVRTQ0NaTkdoMlpDT3Yyallq - eTd0bDViVTgwZGRTUmlYTzR0Y09iWWcKLS0tIEFlQnFPVFRVNlAxdExMekJ2b295 - UUJkUUZCNUZnbkNFZHVBYXNHQklOL2MKujgh6REuAKu6ZLVA7atiWUqhnvYJnQjb - WsxCa9ZXZRgfbhcNlZ3qIKJpWWI/RMS17+Nm5yIl+2cSqe2UJMjZdg== - -----END AGE ENCRYPTED FILE----- - - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNXQ4RVVRd3RYRkhUVExG - SHJON0hwSmJtUkYrd1dldHJRN2tPKzBsNlNjCi9xYVhaanF5TDU2Q0xadXNWR2tN - dHhQVkpRREFlRm5MM2pwVytEaGhHT0EKLS0tIE9sRUtLako3cnAxNm82RDhiWEVM - ZW1IMXkzYkhqbW1ZdVRabUlkK2oxSTgKHC22uQqMq+cJ7vrONkGgoH8snxGef6Ft - QbtoJziERjAhK6B7TOY8AJ3WVRpCzZN70HjLNYa+bMMNOvmlsVxfZg== - -----END AGE ENCRYPTED FILE----- - - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSmZ5YUpFdzRNdWZVNmxJ - bm5ucUhVeTV2TkE4ZElkZ1N3aXc2eVEwMlRNCmxXRElPb2pGYzJFVnUxQkRtMlNF - cjgwUzh5UWNLTk01U0h2bHNpaXVzZkkKLS0tIDczUkkwTG8rL1V3UU9lenk4V2tl - TUxDd2huTllMRG9MZTJZdzRwaWxqUVEKLA3y+heUA8cK31LZzv5A1wtgf+sauuwE - 7SGU3uYU650tJM3e6Lveo+JOAD7Z1jrAomT5Bub+jjSHnpeFC9yMbA== - -----END AGE ENCRYPTED FILE----- - - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4enA2bnlrV3ViY1RHaTVS - Ym5VV005NFlXZUl2NDNXYXBoOHh0SGQ0YVNnCm1KdHBSeE9lQzZEM2hFZUwzRitS - K3BEWGhtWmxKc2RJd0FTMEs2b1ArOUkKLS0tIG5kaWc3U0o4SG9teXk5dVZWWjkz - cS9VMU5YbEl3UE5mODJ1THNLVEdVblEKNQF0b9r1XPD819Z6Uy0b9hT4Uek2tNWU - 3z3H7V/UiB1TMW+qgs6BC6bDkDf7oG//qmZEdYF+lDXcNSwai25xyg== - -----END AGE ENCRYPTED FILE----- - - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBreHJNRXlpOEh6YUxaSmJj - ZDlVdHh2b1p6aEs3eDAvbkk4WExxWmE5bDM0CkZzT2l1K09UbmNFNEpZUVY2NVlB - dVFYbnpvTjlUcTdZejMrelpscXRJQzAKLS0tIEVIaVByVmp0aUU1ZWJLajBhcjRk - QVZMRXBRVVhaY3JKZEJjMTdEeEVqcWMKT+DoevNQAxCrty2VkRDLWGFzs9GsW3F7 - txz73tAceAIiocC1z7IV2TaYULYf7Z75HAje/SOTlGHBIDiVZ0vyLw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqWlk1QWFERzNaZEo2NDhl + cmF6VGNzYnprYXU4ZUsrRnEwK1R0WjFvOEdJCnRQdXExOTVoYlZkSFJqeGQ2QWNQ + T2pkTHdmMDlQczVrd290a0s2MWsvQjQKLS0tIFJVRStURG1vaUFFV2U2THFaazlX + THZPclpjL3FuSlplOUorQVVQYWt4ZTgKla/Ibk00Pz9m3p/E3qVyTWC/6yWzGC0j + bILtkm8rzGx2akXcC/9pIE1+g6Y+x9c7gBJ3aitO7DF7TVgW4DQYUQ== -----END AGE ENCRYPTED FILE----- - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEZEJHQlJqMWxob1lxOUFK - dDZIN3FaNWR1L0gyN3I1MXVXZlpzdlpQUHpjCmIwTWhRamZvSTF3cHZMNk9YUlRv - U2tOK3E5MFBFNERsUHVzVnhsUDFRd00KLS0tIFd1MUpaaFU0bWdVRjJ6NjFwcFZt - bkJGWFFWanFBK1drZlBNcHo0c3Bjc00K/vPBLocRhtcJ3snGYFr+H7qhbg6iSSPP - OSH8WnaM5JmmA9IQlm5uGiG74PHi5sg5d+bwG8pPQtMKN+Ndxh7JIA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPQ1lLcFZTcld3UVpiTWZU + aDZaSFAwN1Nzdlhmam8xTUZNSnpWSHRHM25NCklEUGVUUEg1emVJUHB2TENqY2R1 + Z1ZKNUl4QXd0bGU5bk82Z2YzTnpCWTAKLS0tIFJrWkRvMGxEOWFlQ29rUSs3bkgw + QjJ6V2czTnM0WFFvOGUza0dBalFMdXMKDh65zI+4C/M5u5L8F7ditFxIeAHmNoqu + mAehEmA+iPcnc/+q7aMVnTxsLgbRwrmPpvGKvUaLtPr1pOLpHtvuUw== -----END AGE ENCRYPTED FILE----- - - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0 + - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlN291MzZOaU4zazhEeXBh - WlhoYmh2ZDBsZmc3cEthdW5paWpXbXQvUG1FCjBLZ0FPVWR3T2pVWTZrRmkxSWUr - MHhkUFFPK1Z0b2t1Z1J0VjlER1JvcGMKLS0tIE45YndxVW4vak1wcEJoZzhHQ0E0 - NzA1cy80ZW5vUFplQzVMZ0txSmVkMUEKFUvgmJNdo9sV33gOx7LVUSCYvIqCNwaP - u+XoWTfg4kp9f4KVTy/8huPsVLhZBUaf6jI10mV2z4QwaLHje4JiHw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrQ1NicHlhZFU1eDBqZXVk + SGhma1hSTlpKaEdXNnluK1Z6YkdaNnBvREdVClR6NmpRMFNqYlk2RXEyaWlDejFw + UHR3Q0ZFbGpXOFpDdkRsSytkNDJieEUKLS0tIHpMdGVDWjJPaXpvN2FHZ0VTRUF0 + NmF1a3E5djlXK3MyanRaQUhBc1kyaE0KDAk83Aug7BtVcyzo//EWmN/FYD6pQzSE + 0J5tE5dqkI6VzBrSGzosRsXMhuGcRx6r2XBKJWFINwom9Td87aVCcQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-11-22T05:48:59Z" - mac: ENC[AES256_GCM,data:In/gSIYnXKbbv1lzS/nmSESCHBcBv/TtkvhzdNiIn73N4kP9aJ+1JE8Npix8zNItzk46DX+nHBk8Kwgl6uq26YtL+sMTBKh5K8Ny0H8ivlgS+olXswv3Y9h1cYD7FBHUKzbMuiJd0ppjC0ZIn20rRpb4d57rwUbvY0KstyQW4JA=,iv:DcdTAimbXXpKhhiB9rriS75+XGNOCcScqi/804+Xx6g=,tag:NHW+UViRmbUDHb0gTd9TDg==,type:str] + lastmodified: "2024-11-30T06:31:42Z" + mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1 diff --git a/machines/sops.nix b/machines/sops.nix index 869fef7..c528b95 100644 --- a/machines/sops.nix +++ b/machines/sops.nix @@ -10,20 +10,9 @@ # TODO: How to generate this key when bootstrap? age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; secrets = { - github_public_token = { - owner = "root"; - }; - singbox_sg_server = { - owner = "root"; - }; - singbox_jp_server = { - owner = "root"; - }; - private_dns_address = { - owner = "root"; - }; + "prometheus/metrics_username" = { }; + "prometheus/metrics_password" = { }; }; - secrets.grafana_cloud_api = lib.mkIf config.services.prometheus.enable { owner = "prometheus"; }; }; }; } diff --git a/machines/thorite/default.nix b/machines/thorite/default.nix index 7b7ec7e..b85bab8 100644 --- a/machines/thorite/default.nix +++ b/machines/thorite/default.nix @@ -30,7 +30,6 @@ commonSettings = { auth.enable = true; - autoupgrade.enable = true; }; nixpkgs.system = "x86_64-linux"; diff --git a/machines/thorite/monitoring.nix b/machines/thorite/monitoring.nix index 4f80743..565da59 100644 --- a/machines/thorite/monitoring.nix +++ b/machines/thorite/monitoring.nix @@ -41,6 +41,7 @@ with my-lib; "45.142.178.32:22" "home.xinyang.life:8000" ]; + passwordFile = config.sops.secrets."prometheus/metrics_password".path; in (mkScrapes [ { @@ -50,18 +51,22 @@ with my-lib; port = 8082; } { + inherit passwordFile; name = "gotosocial"; address = "xinyang.life"; } { + inherit passwordFile; name = "miniflux"; address = "rss.xinyang.life"; } { + inherit passwordFile; name = "ntfy"; address = "ntfy.xinyang.life"; } { + inherit passwordFile; name = "grafana-eu"; address = "grafana.xinyang.life"; } diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index b83e212..f96c4c9 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -7,7 +7,7 @@ ./disk-partitions ./restic.nix ./vaultwarden.nix - ./prometheus + ./monitor ./hedgedoc.nix ./sing-box.nix ./kanidm-client.nix diff --git a/modules/nixos/monitor/default.nix b/modules/nixos/monitor/default.nix index e911def..f426d1c 100644 --- a/modules/nixos/monitor/default.nix +++ b/modules/nixos/monitor/default.nix @@ -8,6 +8,7 @@ let mkEnableOption mkOption mkIf + mkMerge types ; cfg = config.custom.prometheus; @@ -58,91 +59,113 @@ in }; }; - config = mkIf cfg.enable { - services.tailscale = { - enable = true; - permitCertUid = config.services.caddy.user; - openFirewall = true; - }; + config = mkMerge [ + { + sops.secrets = { + "prometheus/metrics_username" = { + sopsFile = ../../../machines/secrets.yaml; + group = "prometheus-auth"; + mode = "0440"; + }; - services.caddy = { - enable = true; - virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' - reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} - ''; - }; - services.prometheus = mkIf cfg.enable { - enable = true; - port = 9091; - globalConfig.external_labels = { - hostname = config.networking.hostName; - }; - - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ]; - } - ]; - - alertmanager = { - enable = true; - listenAddress = "127.0.0.1"; - logLevel = "debug"; - configuration = { - route = { - receiver = "ntfy"; - }; - receivers = [ - { - name = "ntfy"; - webhook_configs = [ - { - url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' - Alert {{.status}} - {{range .alerts}}-----{{range $k,$v := .labels}} - {{$k}}={{$v}}{{end}} - {{end}} - ''}"; - send_resolved = true; - } - ]; - } - ]; + "prometheus/metrics_password" = { + sopsFile = ../../../machines/secrets.yaml; + group = "prometheus-auth"; + mode = "0440"; }; }; - alertmanagers = [ + users.groups.prometheus-auth.members = [ + "prometheus" + ]; + } + (mkIf cfg.enable { + + services.tailscale = { + enable = true; + permitCertUid = config.services.caddy.user; + openFirewall = true; + }; + + services.caddy = { + enable = true; + virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' + reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} + ''; + }; + services.prometheus = mkIf cfg.enable { + enable = true; + port = 9091; + globalConfig.external_labels = { + hostname = config.networking.hostName; + }; + + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ]; + } + ]; + + alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + logLevel = "debug"; + configuration = { + route = { + receiver = "ntfy"; + }; + receivers = [ + { + name = "ntfy"; + webhook_configs = [ + { + url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' + Alert {{.status}} + {{range .alerts}}-----{{range $k,$v := .labels}} + {{$k}}={{$v}}{{end}} + {{end}} + ''}"; + send_resolved = true; + } + ]; + } + ]; + }; + }; + + alertmanagers = [ + { + scheme = "http"; + static_configs = [ + { + targets = [ + "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" + ]; + } + ]; + } + ]; + rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ]; + }; + custom.prometheus.ruleModules = [ { - scheme = "http"; - static_configs = [ + name = "prometheus_alerts"; + rules = [ { - targets = [ - "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" - ]; + alert = "JobDown"; + expr = "up == 0"; + for = "1m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "Job {{ $labels.job }} down for 1m."; + }; } ]; } ]; - rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ]; - }; - custom.prometheus.ruleModules = [ - { - name = "prometheus_alerts"; - rules = [ - { - alert = "JobDown"; - expr = "up == 0"; - for = "1m"; - labels = { - severity = "critical"; - }; - annotations = { - summary = "Job {{ $labels.job }} down for 1m."; - }; - } - ]; - } - ]; - }; + }) + ]; } diff --git a/modules/nixos/monitor/exporters.nix b/modules/nixos/monitor/exporters.nix index 7d27f93..0c9b95d 100644 --- a/modules/nixos/monitor/exporters.nix +++ b/modules/nixos/monitor/exporters.nix @@ -46,6 +46,21 @@ in ); }; + # gotosocial + sops.templates."gotosocial_metrics.env" = { + content = '' + GTS_METRICS_AUTH_ENABLED=true + GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"} + GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"} + ''; + group = "prometheus-auth"; + mode = "0440"; + }; + systemd.services.gotosocial.serviceConfig = { + EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ]; + SupplementaryGroups = [ "prometheus-auth" ]; + }; + services.gotosocial.settings = { metrics-enabled = true; }; @@ -55,7 +70,24 @@ in }; services.restic.server.prometheus = true; - systemd.services.miniflux.environment.METRICS_COLLECTOR = "1"; + + # miniflux + sops.templates."miniflux_metrics_env" = { + content = '' + METRICS_COLLECTOR=1 + LOG_LEVEL=debug + METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"} + METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"} + ''; + group = "prometheus-auth"; + mode = "0440"; + }; + + systemd.services.miniflux.serviceConfig = { + EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ]; + SupplementaryGroups = [ "prometheus-auth" ]; + }; + services.ntfy-sh.settings.enable-metrics = true; services.caddy.globalConfig = '' diff --git a/overlays/my-lib/prometheus.nix b/overlays/my-lib/prometheus.nix index 29a0362..e4c87cd 100644 --- a/overlays/my-lib/prometheus.nix +++ b/overlays/my-lib/prometheus.nix @@ -7,6 +7,7 @@ in { name, address, + passwordFile ? null, port ? 443, scheme ? "https", ... @@ -16,6 +17,15 @@ in scheme = scheme; static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ]; } + // ( + if isNull null then + { } + else + { + basic_auth.username = "prom"; + basic_auth.password_file = passwordFile; + } + ) ); mkCaddyScrapes = mkFunction (