diff --git a/.sops.yaml b/.sops.yaml index 5056c87..dded97c 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -2,13 +2,22 @@ keys: - &xin age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c - &host-calcite age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa - &host-raspite age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj - - &host-sgp-00 age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx - &host-la-00 age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh - &host-massicot age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta - &host-weilite age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml - &host-hk-00 age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9 - &host-fra-00 age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s + - &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv + - &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96 creation_rules: + - path_regex: machines/secrets.yaml + key_groups: + - age: + - *xin + - *host-calcite + - *host-massicot + - *host-thorite + - *host-biotite - path_regex: machines/calcite/secrets.yaml key_groups: - age: @@ -24,19 +33,23 @@ creation_rules: - age: - *xin - *host-massicot + - paht_regex: machines/biotite/secrets.yaml + key_groups: + - age: + - *xin + - *host-biotite + - path_regex: machines/thorite/secrets.yaml + key_groups: + - age: + - *xin + - *host-thorite - path_regex: machines/dolomite/secrets/secrets.yaml key_groups: - age: - *xin - - *host-sgp-00 - *host-la-00 - *host-hk-00 - *host-fra-00 - - path_regex: machines/dolomite/secrets/sgp-00.yaml - key_groups: - - age: - - *xin - - *host-sgp-00 - path_regex: machines/dolomite/secrets/la-00.yaml key_groups: - age: @@ -58,16 +71,6 @@ creation_rules: - age: - *xin - *host-weilite - - path_regex: machines/secrets.yaml - key_groups: - - age: - - *xin - - *host-calcite - - *host-raspite - - *host-sgp-00 - - *host-la-00 - - *host-hk-00 - - *host-massicot - path_regex: home/xin/secrets.yaml key_groups: - age: diff --git a/flake.nix b/flake.nix index 55a8561..7e725f2 100644 --- a/flake.nix +++ b/flake.nix @@ -60,8 +60,8 @@ outputs = { self, - home-manager, nixpkgs, + home-manager, nixos-hardware, sops-nix, flake-utils, @@ -84,6 +84,7 @@ overlayModule = { ... }: { + _module.args.my-lib = import ./overlays/my-lib; nixpkgs.overlays = [ editorOverlay (import ./overlays/add-pkgs.nix) @@ -118,10 +119,6 @@ ./machines/dolomite/bandwagon.nix ./machines/dolomite/common.nix ]; - tok-00 = [ - ./machines/dolomite/lightsail.nix - ./machines/dolomite/common.nix - ]; fra-00 = [ ./machines/dolomite/fra.nix ./machines/dolomite/common.nix @@ -178,7 +175,6 @@ }; in { - nixpkgs = nixpkgs; nixosModules.default = { imports = [ ./modules/nixos @@ -190,7 +186,9 @@ colmenaHive = colmena.lib.makeHive { meta = { # FIXME: - nixpkgs = import nixpkgs { system = "x86_64-linux"; }; + nixpkgs = import nixpkgs { + system = "x86_64-linux"; + }; }; massicot = @@ -205,20 +203,6 @@ ] ++ sharedColmenaModules; }; - tok-00 = - { ... }: - { - imports = nodeNixosModules.tok-00 ++ sharedColmenaModules; - nixpkgs.system = "x86_64-linux"; - networking.hostName = "tok-00"; - system.stateVersion = "23.11"; - deployment = { - targetHost = "video01.namely.icu"; - buildOnTarget = false; - tags = [ "proxy" ]; - }; - }; - la-00 = { ... }: { @@ -310,7 +294,6 @@ osmium = mkNixos { hostname = "osmium"; }; - } // self.colmenaHive.nodes; } diff --git a/machines/biotite/default.nix b/machines/biotite/default.nix index 1b73ee4..5021dc8 100644 --- a/machines/biotite/default.nix +++ b/machines/biotite/default.nix @@ -12,11 +12,12 @@ networking.useNetworkd = true; systemd.network.enable = true; systemd.network.networks."10-wan" = { - matchConfig.MACAddress = "00:16:3e:0a:ec:45"; - networkConfig.DHCP = "ipv4"; - dhcpV4Config = { - UseDNS = true; + matchConfig.MACAddress = "b6:20:0d:9a:6c:34"; + networkConfig = { + DHCP = "ipv4"; + IPv6SendRA = true; }; + address = [ "2a03:4000:4a:148::1/64" ]; }; commonSettings = { diff --git a/machines/calcite/configuration.nix b/machines/calcite/configuration.nix index 1be4864..181c81f 100644 --- a/machines/calcite/configuration.nix +++ b/machines/calcite/configuration.nix @@ -368,8 +368,7 @@ in }; custom.prometheus = { - enable = true; - exporters.blackbox.enable = true; + exporters.node.enable = true; }; services.ollama = { diff --git a/machines/dolomite/claw.nix b/machines/dolomite/claw.nix index d169733..f7b64b7 100644 --- a/machines/dolomite/claw.nix +++ b/machines/dolomite/claw.nix @@ -31,14 +31,7 @@ fsType = "ext4"; }; - swapDevices = [ ]; - - # Enables DHCP on each ethernet and wireless interface. In case of scripted networking - # (the default) this is the recommended approach. When using systemd-networkd it's - # still possible to use this option, but it's recommended to use it in conjunction - # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - # networking.useNetworkd = false; - + networking.useNetworkd = true; systemd.network.enable = true; systemd.network.networks."10-wan" = { matchConfig.MACAddress = "00:16:3e:0a:ec:45"; @@ -47,7 +40,6 @@ UseDNS = true; }; }; - # networking.interfaces.eth0.useDHCP = lib.mkDefault true; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; }; diff --git a/machines/dolomite/common.nix b/machines/dolomite/common.nix index 3b511ef..23306c0 100644 --- a/machines/dolomite/common.nix +++ b/machines/dolomite/common.nix @@ -21,12 +21,20 @@ }; }; }; + swapDevices = [ + { + device = "/swapfile"; + size = 2 * 1024; + } + ]; - custom.prometheus = { - enable = lib.mkDefault true; - exporters.blackbox.enable = true; + custom.prometheus.exporters = { + enable = true; + node.enable = true; }; + services.tailscale.enable = true; + commonSettings = { auth.enable = true; proxyServer = { diff --git a/machines/dolomite/fra.nix b/machines/dolomite/fra.nix index 0caf650..c5a8d02 100644 --- a/machines/dolomite/fra.nix +++ b/machines/dolomite/fra.nix @@ -52,6 +52,12 @@ networkConfig = { DHCP = "no"; Gateway = "185.217.108.1"; + DNSSEC = true; + DNSOverTLS = true; + DNS = [ + "8.8.8.8#dns.google" + "8.8.4.4#dns.google" + ]; }; address = [ "185.217.108.59/24" ]; }; diff --git a/machines/massicot/default.nix b/machines/massicot/default.nix index e461039..da2cbd5 100644 --- a/machines/massicot/default.nix +++ b/machines/massicot/default.nix @@ -24,10 +24,6 @@ hedgedoc_env = { owner = "hedgedoc"; }; - grafana_cloud_api = { - owner = "prometheus"; - sopsFile = ../secrets.yaml; - }; grafana_oauth_secret = { owner = "grafana"; }; @@ -62,6 +58,8 @@ hostName = "massicot"; }; + services.tailscale.enable = true; + commonSettings = { auth.enable = true; nix = { diff --git a/machines/massicot/kanidm-provision.nix b/machines/massicot/kanidm-provision.nix index ef8323b..8a95a99 100644 --- a/machines/massicot/kanidm-provision.nix +++ b/machines/massicot/kanidm-provision.nix @@ -108,6 +108,20 @@ ]; }; }; + gotosocial = { + displayName = "GoToSocial"; + originUrl = "https://gts.xiny.li/auth/callback"; + originLanding = "https://gts.xiny.li/auth/callback"; + allowInsecureClientDisablePkce = true; + scopeMaps = { + gts-users = [ + "openid" + "email" + "profile" + "groups" + ]; + }; + }; # It's used for all the clients. I'm too lazy to change the name. owncloud-android = { displayName = "ownCloud Apps"; @@ -147,7 +161,7 @@ immich = { displayName = "Immich"; originUrl = [ - "https://immich.xinyang.life:8000/api/oauth/mobile-redirect" + "https://immich.xinyang.life:8000/api/oauth/mobile-redirect/" "https://immich.xinyang.life:8000/auth/login" "https://immich.xinyang.life:8000/user-settings" ]; diff --git a/machines/massicot/networking.nix b/machines/massicot/networking.nix index 7859b2e..2a4c529 100644 --- a/machines/massicot/networking.nix +++ b/machines/massicot/networking.nix @@ -3,8 +3,10 @@ networking.useNetworkd = true; systemd.network.networks."10-wan" = { matchConfig.MACAddress = "96:00:02:68:7d:2d"; - networkConfig.DHCP = "ipv4"; - networkConfig.Gateway = "fe80::1"; + networkConfig = { + DHCP = "ipv4"; + Gateway = "fe80::1"; + }; address = [ "2a01:4f8:c17:345f::3/64" ]; diff --git a/machines/massicot/services.nix b/machines/massicot/services.nix index 6a43aa3..a1e69a0 100644 --- a/machines/massicot/services.nix +++ b/machines/massicot/services.nix @@ -43,10 +43,14 @@ in environmentFile = config.sops.secrets.hedgedoc_env.path; }; - custom.prometheus = { + custom.prometheus.exporters = { enable = true; - exporters.blackbox.enable = true; - exporters.miniflux.enable = true; + blackbox = { + enable = true; + }; + node = { + enable = true; + }; }; security.acme = { diff --git a/machines/netdrives.nix b/machines/netdrives.nix deleted file mode 100644 index 2fedf53..0000000 --- a/machines/netdrives.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs, config, ... }: -{ - sops.secrets = { - autofs-nas = { - owner = "davfs2"; - }; - autofs-nas-secret = { - path = "/etc/davfs2/secrets"; - }; - }; - fileSystems."/media/nas" = { - device = "https://home.xinyang.life:5244/dav"; - fsType = "davfs"; - options = [ - "uid=1000" - "gid=1000" - "rw" - "_netdev" - ]; - - }; -} diff --git a/machines/secrets.yaml b/machines/secrets.yaml index cedd676..e179455 100644 --- a/machines/secrets.yaml +++ b/machines/secrets.yaml @@ -1,14 +1,6 @@ -clash_subscription_link: ENC[AES256_GCM,data:uDaX2BE/qRdfXVtckX0VKpu0LN3j0YaxVIPbQt3tGAfdfqFqlp0IzFgNiZBIEcIltYkeEyqFSA0QnttoMb0QYe9f2rtgjztwk10SOGViGaeFWPfkdlHP04qhm5OOOddi3OwT5rUNwvBU79AdCnLJ9QwqMbOaNm/JTtbkcjf8huxc2UcYAQcY/YNJ7aTEhWIw98Ab85aih+w=,iv:pZ189IPPCBjscXzEdgQCRdFlls3TniwDfNCd+H1FFaQ=,tag:dMmGZvppWtkc82b5dTnJwg==,type:str] -autofs-nas: ENC[AES256_GCM,data:LnCKGKARx6Vd99VwAX/6PXOJwo+a7GP8fNmM9yuuC2xITGxtWCsDdOZL1+IA5LS/gbOYINgQWDzWirJF3LCP27BQeLwXYpD7/UAwwVI=,iv:QJzsS5a6vWeoBxkB13yXdVbyn0tt2QTvqj0LaHn6S2g=,tag:D/JKXQIw1EzIh3wjGhHgHg==,type:str] -autofs-nas-secret: ENC[AES256_GCM,data:gbOizRZAvh79HlJWIWeKTk79Ux311XGL1eIswc0P2U2huCibD/ji3kOlSjZXENG+fJQKNz2AlDTk3g2cQQ==,iv:UCaGeE8j4RqJzA0xhu3oB2xvzombzQD3fjLKCWd5fDg=,tag:II4eEMr7f2TDUl1qUcDYXA==,type:str] -github_public_token: ENC[AES256_GCM,data:6Gt+oJcCRHeoLK7CRndMMbszTXSEbnN0nQzsVOnl/+zB4hxbEPD5k/vkkl+cZ/qmxdxFXV0OOsYvktn44Yv1DMUE3mkB0hcAdoyPwLuYM7W3RpOoW3OktH8DRCUi6msvFp3ykpdmIl9WyjVhc/lMwTaYJQyRh1ue,iv:PJSFtJBelyc3rzd6hqjMp+ciU2Q3FTOEXsiq5F2KKTY=,tag:Y/stRg6kwyjjIFZCXS/peg==,type:str] -singbox_sg_server: ENC[AES256_GCM,data:SF2ja6W4TwThwoug5x2KTA==,iv:Vx9wNTdVHkReux4YeQY+0VkC1Wqg/CRkY7frVY/3e50=,tag:7XA9KSoR0GA6FoYRhCv4BQ==,type:str] -singbox_jp_server: ENC[AES256_GCM,data:S3Bs5yVMzyz6vD51GYElOM5h,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:o9d55cZuWmX4NDYexWjvYQ==,type:str] -sing-box: - password: ENC[AES256_GCM,data:xyqmoJEDI5959zHPTVelln/iThtoeDwS,iv:rLyqJsE/4JDf08RlMLLPh+MKJkba9bL0z8jx6bTEfgc=,tag:cgLHdeLIyPvLhRNaVcQ0TQ==,type:str] - uuid: ENC[AES256_GCM,data:lWBCM5wyz6BcUUHdvynkn5y166Kk15jO0EhWUDuhXXhrve5l,iv:RmDJYFnYqIEIShLn25sf4h8AO2E3+3Xa2U9Mff+Xk2w=,tag:SN0DUdwZXKO/VEnozrr5mA==,type:str] -grafana_cloud_api: ENC[AES256_GCM,data:eEvPAwtThK1FMhbrnmSo89+GlWZAF+LQRMLXA2C6f1vR7ZPlXJZGWzjYwDcPlnpiC737/cG14M4kZqvPGBuNub5A83rBS/+FeebvGDIF59L5PC1Ys1jWBB9YRI/L9EU0tvwTTUCvLRA9j28n7Jw7wR6mWXm63XA+OMu8/UbTwbeV/WUQn8vnwqadSUdCnNKJXMsAY+q9t/st0DPm5+aNxA==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:87C+0FVvzDIowE0+QpY1zA==,type:str] -private_dns_address: ENC[AES256_GCM,data:YJxNOH4hsZHResvANEqJRTANhnL4PLp/Pmi/PhgtSTbTKiJKPqudhTEkNg==,iv:8+qG5rQXAKfrykEjt9qrbtyNaBuKvi7EaIWouRqEipY=,tag:VH0w5ZbXcWFGZ9GLavm7/w==,type:str] +prometheus: + metrics_username: ENC[AES256_GCM,data:/CQfOA==,iv:BjhB+uLfjmYHdgpc/+tDJXJ8C1EK9kngQWbo4NleOmE=,tag:JCdqyqGLRh09T25vmufiZw==,type:str] + metrics_password: ENC[AES256_GCM,data:q/xMPuNtlcUFewMdVu6w2Q==,iv:xLohdb5tdxevYFckZoacjSJp2rZ53QKLxK6u3mc3mDw=,tag:B4LrObH1DsnnD5CcuOPOyg==,type:str] sops: kms: [] gcp_kms: [] @@ -18,77 +10,50 @@ sops: - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiRzZVNGFocUN2VzZLTmJz - WlJnUmxhZS92citDRkVZVnJZQU9YWVZORlNjCkgzeWl5dTl1YmpjZGt2anF3dGgr - K1hOSTRmakNrZ2JoNit3NDIzK1FCcWMKLS0tIEdqY3VvR3gxd1JoQlhPR3JvcXBF - K2g4VFpqUEF5RTQ3cmpUSG0xajN2bUUKMuwx5cO1nHokV1NOloXfl9wTBN/+/Rlq - UJKP/qaI23tpyMXN1U40iF20ecO1U5Ad8wAQ61C/tldSVULizDihpw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmUUtKcU0va1F3SjFzcktl + S0FJNHlTUzBkMWlydGRjM1Y4SGd2SXpMVTJRCmtRSW9wNW9xMDBaQ0YzUWM4YjRz + SVRDNHRjNG5hTHBOOHorTTlJU1BwY1EKLS0tIEpLREJ1VzFaalczZlhKaitHVTJU + MDdJaVBtVmw4WTlBUEF5WXJSVFRFeDAKnvF6CmnU8hxXSdKQPUJqPT7Dewl4REOH + wDQELRaDkMPMKEOAc6wCmXNErvj/I7w7wuvB5WxtanC7g4IEphD6aA== -----END AGE ENCRYPTED FILE----- - recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYXlwdytVSm1SQzRubHdX - dHhrTWxyamo3OFRraEVRQ3plK1cwUWt0a1JRCkdqaVRTQ0NaTkdoMlpDT3Yyallq - eTd0bDViVTgwZGRTUmlYTzR0Y09iWWcKLS0tIEFlQnFPVFRVNlAxdExMekJ2b295 - UUJkUUZCNUZnbkNFZHVBYXNHQklOL2MKujgh6REuAKu6ZLVA7atiWUqhnvYJnQjb - WsxCa9ZXZRgfbhcNlZ3qIKJpWWI/RMS17+Nm5yIl+2cSqe2UJMjZdg== - -----END AGE ENCRYPTED FILE----- - - recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNXQ4RVVRd3RYRkhUVExG - SHJON0hwSmJtUkYrd1dldHJRN2tPKzBsNlNjCi9xYVhaanF5TDU2Q0xadXNWR2tN - dHhQVkpRREFlRm5MM2pwVytEaGhHT0EKLS0tIE9sRUtLako3cnAxNm82RDhiWEVM - ZW1IMXkzYkhqbW1ZdVRabUlkK2oxSTgKHC22uQqMq+cJ7vrONkGgoH8snxGef6Ft - QbtoJziERjAhK6B7TOY8AJ3WVRpCzZN70HjLNYa+bMMNOvmlsVxfZg== - -----END AGE ENCRYPTED FILE----- - - recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSmZ5YUpFdzRNdWZVNmxJ - bm5ucUhVeTV2TkE4ZElkZ1N3aXc2eVEwMlRNCmxXRElPb2pGYzJFVnUxQkRtMlNF - cjgwUzh5UWNLTk01U0h2bHNpaXVzZkkKLS0tIDczUkkwTG8rL1V3UU9lenk4V2tl - TUxDd2huTllMRG9MZTJZdzRwaWxqUVEKLA3y+heUA8cK31LZzv5A1wtgf+sauuwE - 7SGU3uYU650tJM3e6Lveo+JOAD7Z1jrAomT5Bub+jjSHnpeFC9yMbA== - -----END AGE ENCRYPTED FILE----- - - recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4enA2bnlrV3ViY1RHaTVS - Ym5VV005NFlXZUl2NDNXYXBoOHh0SGQ0YVNnCm1KdHBSeE9lQzZEM2hFZUwzRitS - K3BEWGhtWmxKc2RJd0FTMEs2b1ArOUkKLS0tIG5kaWc3U0o4SG9teXk5dVZWWjkz - cS9VMU5YbEl3UE5mODJ1THNLVEdVblEKNQF0b9r1XPD819Z6Uy0b9hT4Uek2tNWU - 3z3H7V/UiB1TMW+qgs6BC6bDkDf7oG//qmZEdYF+lDXcNSwai25xyg== - -----END AGE ENCRYPTED FILE----- - - recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBreHJNRXlpOEh6YUxaSmJj - ZDlVdHh2b1p6aEs3eDAvbkk4WExxWmE5bDM0CkZzT2l1K09UbmNFNEpZUVY2NVlB - dVFYbnpvTjlUcTdZejMrelpscXRJQzAKLS0tIEVIaVByVmp0aUU1ZWJLajBhcjRk - QVZMRXBRVVhaY3JKZEJjMTdEeEVqcWMKT+DoevNQAxCrty2VkRDLWGFzs9GsW3F7 - txz73tAceAIiocC1z7IV2TaYULYf7Z75HAje/SOTlGHBIDiVZ0vyLw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBS2xqbWp0NkFMVm1Cci82 + d0ZmRXdRc2FFSklHNnZtV1Z0REFIelZDQ1U4CllmdllNVnp3WmpCeDNBRzVFbVR5 + WkJFMGs5ZWJEK1lSWDQwYUdOdFJseGcKLS0tIEZUU254aWtYdWthL3I2UkJ0eklj + WHhrRlRvLzlmY0REYktGSlh3MENzRzgKzO1XqXhcXAxfn86+IY+ccBII1SGYctAk + +ArpGmXaf53RFmPLSzMGNaiJzfhqk9U9bn3WV9CFdaA7Rtec0ZAcNw== -----END AGE ENCRYPTED FILE----- - recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEZEJHQlJqMWxob1lxOUFK - dDZIN3FaNWR1L0gyN3I1MXVXZlpzdlpQUHpjCmIwTWhRamZvSTF3cHZMNk9YUlRv - U2tOK3E5MFBFNERsUHVzVnhsUDFRd00KLS0tIFd1MUpaaFU0bWdVRjJ6NjFwcFZt - bkJGWFFWanFBK1drZlBNcHo0c3Bjc00K/vPBLocRhtcJ3snGYFr+H7qhbg6iSSPP - OSH8WnaM5JmmA9IQlm5uGiG74PHi5sg5d+bwG8pPQtMKN+Ndxh7JIA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycmppSGR1YTMzTTJQSmx0 + eFlRQWpPRTR1L042MXlqd3dURStjaUZjR1M4Ci9VdzZkSmN3d1BDNTlJdXNSenhZ + MDE1VktBN252L2FYMjJmNEFITGptM3MKLS0tIDNYOEZqNjM1VCtEWDlGdzYveG5j + dEp2bmVQMmV5ZU9Jb3FFSDFoT2NJOE0Kx1ZifyU2WLoHeUmqP9oCUmIl6ZJeytGB + WPMJKcNtuJHL1OWhT0wMiv6NEF5UaYXIlCqSVtXAMy554G4JlX5tQw== -----END AGE ENCRYPTED FILE----- - - recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0 + - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlN291MzZOaU4zazhEeXBh - WlhoYmh2ZDBsZmc3cEthdW5paWpXbXQvUG1FCjBLZ0FPVWR3T2pVWTZrRmkxSWUr - MHhkUFFPK1Z0b2t1Z1J0VjlER1JvcGMKLS0tIE45YndxVW4vak1wcEJoZzhHQ0E0 - NzA1cy80ZW5vUFplQzVMZ0txSmVkMUEKFUvgmJNdo9sV33gOx7LVUSCYvIqCNwaP - u+XoWTfg4kp9f4KVTy/8huPsVLhZBUaf6jI10mV2z4QwaLHje4JiHw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5OGlRWVM0Z01pYjdqYjEy + TGdhNFhpNEUwSFZaeHNwRkpraFF5RTU2SXhnCm5YWmM5SmdERzZBWTgva1E4MDFm + N0xyUExGV0MvbFF3M0ZRSVEydFNUSGMKLS0tIGxxNWhsSEt4WDR2a2hId1JkVFE3 + enJ6MzJxR0I4eStSQk9ON0dsdjFmRkEKBSGkv1O0vgHSsU3+6AGN7bKQ5lpN7AMT + eqEgWx7juZ7hKzLq1HMbiT61l0FrJNHEMfn15bzn7GsK5YJQvfiq9w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-11-22T05:48:59Z" - mac: ENC[AES256_GCM,data:In/gSIYnXKbbv1lzS/nmSESCHBcBv/TtkvhzdNiIn73N4kP9aJ+1JE8Npix8zNItzk46DX+nHBk8Kwgl6uq26YtL+sMTBKh5K8Ny0H8ivlgS+olXswv3Y9h1cYD7FBHUKzbMuiJd0ppjC0ZIn20rRpb4d57rwUbvY0KstyQW4JA=,iv:DcdTAimbXXpKhhiB9rriS75+XGNOCcScqi/804+Xx6g=,tag:NHW+UViRmbUDHb0gTd9TDg==,type:str] + - recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeVlxRk1PZUpUVm1xckRu + ZmMzdHdvKzExaTQxVHYvYjIvblo3b1ZORFJrCkpVdHFLbCtNS0xnamJ0T250YzUy + Uy9Xd0tMa3FSVlRkQXFaTWJVem9uWGsKLS0tIFRmT0VzL0hlLzkrRTZxcWtLN3Qv + YVMya3dUazFyaWRNNDJ3OVNIVXJLVTQK+7MxkmBjPszozXUO+zVaWdsovDmhWAfz + 8puIpXpWZY09BkS0vs4oNhiVA9PD11TBIVCEbC5E1TwpwboMXBYhCQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-11-30T06:31:42Z" + mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1 diff --git a/machines/sops.nix b/machines/sops.nix index 869fef7..c528b95 100644 --- a/machines/sops.nix +++ b/machines/sops.nix @@ -10,20 +10,9 @@ # TODO: How to generate this key when bootstrap? age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; secrets = { - github_public_token = { - owner = "root"; - }; - singbox_sg_server = { - owner = "root"; - }; - singbox_jp_server = { - owner = "root"; - }; - private_dns_address = { - owner = "root"; - }; + "prometheus/metrics_username" = { }; + "prometheus/metrics_password" = { }; }; - secrets.grafana_cloud_api = lib.mkIf config.services.prometheus.enable { owner = "prometheus"; }; }; }; } diff --git a/machines/thorite/default.nix b/machines/thorite/default.nix index ef69751..b85bab8 100644 --- a/machines/thorite/default.nix +++ b/machines/thorite/default.nix @@ -1,29 +1,40 @@ -{ ... }: { imports = [ ./hardware-configurations.nix + ./monitoring.nix ]; - networking.hostName = "thorite"; - networking.useNetworkd = true; - systemd.network.enable = true; - systemd.network.networks."10-wan" = { - matchConfig.MACAddress = "00:51:d3:21:f3:28"; - networkConfig = { - DHCP = "no"; - Gateway = "23.165.200.1"; + config = { + networking.hostName = "thorite"; + networking.useNetworkd = true; + systemd.network.enable = true; + systemd.network.networks."10-wan" = { + matchConfig.MACAddress = "00:51:d3:21:f3:28"; + networkConfig = { + DHCP = "no"; + Gateway = "23.165.200.1"; + DNSSEC = true; + DNSOverTLS = true; + DNS = [ + "8.8.8.8#dns.google" + "8.8.4.4#dns.google" + ]; + }; + address = [ "23.165.200.99/24" ]; }; - address = [ "23.165.200.99/24" ]; + + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + + commonSettings = { + auth.enable = true; + }; + + nixpkgs.system = "x86_64-linux"; + system.stateVersion = "24.11"; + + users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU."; }; - - nixpkgs.system = "x86_64-linux"; - - system.stateVersion = "24.11"; - - commonSettings = { - auth.enable = true; - autoupgrade.enable = true; - }; - - users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU."; } diff --git a/machines/thorite/monitoring.nix b/machines/thorite/monitoring.nix new file mode 100644 index 0000000..2f2b685 --- /dev/null +++ b/machines/thorite/monitoring.nix @@ -0,0 +1,103 @@ +{ config, my-lib, ... }: +with my-lib; +{ + config = { + sops = { + defaultSopsFile = ./secrets.yaml; + age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + secrets = { + "grafana/oauth_secret" = { + owner = "grafana"; + }; + }; + }; + + custom.monitoring = { + grafana.enable = true; + }; + + services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig = + with config.services.grafana.settings.server; '' + reverse_proxy http://${http_addr}:${toString http_port} + ''; + + custom.prometheus = { + enable = true; + exporters = { + enable = true; + blackbox.enable = true; + node.enable = true; + }; + ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]); + }; + + services.prometheus.scrapeConfigs = + let + probeList = [ + "la-00.video.namely.icu:8080" + "fre-00.video.namely.icu:8080" + "hk-00.video.namely.icu:8080" + "49.13.13.122:443" + "45.142.178.32:22" + "home.xinyang.life:8000" + ]; + passwordFile = config.sops.secrets."prometheus/metrics_password".path; + in + (mkScrapes [ + { + name = "immich"; + scheme = "http"; + address = "weilite.coho-tet.ts.net"; + port = 8082; + } + { + inherit passwordFile; + name = "gotosocial"; + address = "xinyang.life"; + } + { + inherit passwordFile; + name = "miniflux"; + address = "rss.xinyang.life"; + } + { + name = "ntfy"; + address = "ntfy.xinyang.life"; + } + { + name = "grafana-eu"; + address = "grafana.xinyang.life"; + } + ]) + ++ (mkCaddyScrapes [ + { address = "thorite.coho-tet.ts.net"; } + ]) + ++ (mkNodeScrapes [ + { address = "thorite.coho-tet.ts.net"; } + { address = "massicot.coho-tet.ts.net"; } + { address = "weilite.coho-tet.ts.net"; } + { address = "hk-00.coho-tet.ts.net"; } + { address = "la-00.coho-tet.ts.net"; } + { address = "fra-00.coho-tet.ts.net"; } + ]) + ++ (mkBlackboxScrapes [ + { + hostAddress = "thorite.coho-tet.ts.net"; + targetAddresses = probeList; + } + { + hostAddress = "massicot.coho-tet.ts.net"; + targetAddresses = probeList; + } + { + hostAddress = "weilite.coho-tet.ts.net"; + targetAddresses = [ + "la-00.video.namely.icu:8080" + "fre-00.video.namely.icu:8080" + "hk-00.video.namely.icu:8080" + ]; + } + ]); + + }; +} diff --git a/machines/thorite/secrets.yaml b/machines/thorite/secrets.yaml new file mode 100644 index 0000000..60d475f --- /dev/null +++ b/machines/thorite/secrets.yaml @@ -0,0 +1,31 @@ +grafana: + oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoTXRtTlRES3V4MGhZaGdr + aXJ4UFNDT0Nrb0ZuWEkxUEFDU2orbzNBSVhVCkh2VitqMGwwOVdhMFJIeWU1eTgw + UVdxY0tLVDJNVnRnQmMyS0FPYS9LVmMKLS0tIEZaMTdIMU5SQUkxL2NFK2Jtbm9v + YVR3RHpDR3F2aFlCWGd5TjNOV2p4YzgK8OKpwcvTK/0j+kQCo0+8n6sQ5Pu9t9xZ + lPWeUGk1BudsyCqgIZWF5iXfu1pJnYq1XEAM0ttJl402xKeqIovM0Q== + -----END AGE ENCRYPTED FILE----- + - recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIYk1NTlhsYW8xbFppNTBE + WlJmNzhnclBoVENXa1cvcHY5NGdRZVAzV0FJClpsTHpTeG9CK3J2ZFEreG1BTWpG + WjdaYzlLQnU0LzJLSDBZZ2pvOWdvSEkKLS0tIExRT0p1aCttZG5MMW14emJmRk5w + M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx + mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-11-28T17:02:03Z" + mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.1 diff --git a/machines/weilite/default.nix b/machines/weilite/default.nix index 8a58896..b2c761d 100644 --- a/machines/weilite/default.nix +++ b/machines/weilite/default.nix @@ -16,7 +16,6 @@ networking.hostName = "weilite"; commonSettings = { auth.enable = true; - autoupgrade.enable = true; nix = { enable = true; enableMirrors = true; @@ -61,8 +60,14 @@ }; }; - custom.prometheus = { + custom.prometheus.exporters = { enable = true; + blackbox = { + enable = true; + }; + node = { + enable = true; + }; }; systemd.mounts = [ diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index b83e212..f96c4c9 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -7,7 +7,7 @@ ./disk-partitions ./restic.nix ./vaultwarden.nix - ./prometheus + ./monitor ./hedgedoc.nix ./sing-box.nix ./kanidm-client.nix diff --git a/modules/nixos/monitor/default.nix b/modules/nixos/monitor/default.nix new file mode 100644 index 0000000..f426d1c --- /dev/null +++ b/modules/nixos/monitor/default.nix @@ -0,0 +1,171 @@ +{ + config, + lib, + ... +}: +let + inherit (lib) + mkEnableOption + mkOption + mkIf + mkMerge + types + ; + cfg = config.custom.prometheus; + + mkRulesOption = mkOption { + type = types.listOf ( + types.submodule { + options = { + name = mkOption { type = lib.types.str; }; + rules = mkOption { type = lib.types.listOf lib.types.attrs; }; + }; + } + ); + }; +in +{ + imports = [ + ./exporters.nix + ./grafana.nix + ]; + + options = { + custom.monitoring = { + grafana = { + enable = mkEnableOption "grafana with oauth only"; + }; + }; + custom.prometheus = { + enable = mkEnableOption "Prometheus instance"; + ruleModules = mkRulesOption; + exporters = { + enable = mkEnableOption "prometheus exporter on all supported and enable guarded services"; + node = { + enable = mkEnableOption "node exporter"; + listenAddress = mkOption { + type = types.str; + default = "${config.networking.hostName}.coho-tet.ts.net"; + }; + }; + blackbox = { + enable = mkEnableOption "blackbox exporter"; + listenAddress = mkOption { + type = types.str; + default = "${config.networking.hostName}.coho-tet.ts.net"; + }; + }; + }; + }; + }; + + config = mkMerge [ + { + sops.secrets = { + "prometheus/metrics_username" = { + sopsFile = ../../../machines/secrets.yaml; + group = "prometheus-auth"; + mode = "0440"; + }; + + "prometheus/metrics_password" = { + sopsFile = ../../../machines/secrets.yaml; + group = "prometheus-auth"; + mode = "0440"; + }; + }; + + users.groups.prometheus-auth.members = [ + "prometheus" + ]; + } + (mkIf cfg.enable { + + services.tailscale = { + enable = true; + permitCertUid = config.services.caddy.user; + openFirewall = true; + }; + + services.caddy = { + enable = true; + virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' + reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} + ''; + }; + services.prometheus = mkIf cfg.enable { + enable = true; + port = 9091; + globalConfig.external_labels = { + hostname = config.networking.hostName; + }; + + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ]; + } + ]; + + alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + logLevel = "debug"; + configuration = { + route = { + receiver = "ntfy"; + }; + receivers = [ + { + name = "ntfy"; + webhook_configs = [ + { + url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' + Alert {{.status}} + {{range .alerts}}-----{{range $k,$v := .labels}} + {{$k}}={{$v}}{{end}} + {{end}} + ''}"; + send_resolved = true; + } + ]; + } + ]; + }; + }; + + alertmanagers = [ + { + scheme = "http"; + static_configs = [ + { + targets = [ + "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" + ]; + } + ]; + } + ]; + rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ]; + }; + custom.prometheus.ruleModules = [ + { + name = "prometheus_alerts"; + rules = [ + { + alert = "JobDown"; + expr = "up == 0"; + for = "1m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "Job {{ $labels.job }} down for 1m."; + }; + } + ]; + } + ]; + }) + ]; +} diff --git a/modules/nixos/monitor/exporters.nix b/modules/nixos/monitor/exporters.nix new file mode 100644 index 0000000..0c9b95d --- /dev/null +++ b/modules/nixos/monitor/exporters.nix @@ -0,0 +1,102 @@ +{ + config, + pkgs, + lib, + ... +}: +let + inherit (lib) mkIf; + cfg = config.custom.prometheus.exporters; +in +{ + config = { + systemd.services.tailscaled.after = + (lib.optional cfg.node.enable "prometheus-node-exporters.service") + ++ (lib.optional cfg.blackbox.enable "prometheus-blackbox-exporters.service") + ++ (lib.optional config.services.caddy.enable "caddy.service"); + + services.prometheus.exporters.node = mkIf cfg.node.enable { + enable = true; + enabledCollectors = [ + "loadavg" + "time" + "systemd" + ]; + listenAddress = cfg.node.listenAddress; + port = 9100; + }; + + services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable { + enable = true; + listenAddress = cfg.blackbox.listenAddress; + configFile = pkgs.writeText "blackbox.config.yaml" ( + lib.generators.toYAML { } { + modules = { + tcp4_connect = { + prober = "tcp"; + tcp = { + ip_protocol_fallback = false; + preferred_ip_protocol = "ip4"; + tls = false; + }; + timeout = "15s"; + }; + }; + } + ); + }; + + # gotosocial + sops.templates."gotosocial_metrics.env" = { + content = '' + GTS_METRICS_AUTH_ENABLED=true + GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"} + GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"} + ''; + group = "prometheus-auth"; + mode = "0440"; + }; + systemd.services.gotosocial.serviceConfig = { + EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ]; + SupplementaryGroups = [ "prometheus-auth" ]; + }; + + services.gotosocial.settings = { + metrics-enabled = true; + }; + + services.immich.environment = { + IMMICH_TELEMETRY_INCLUDE = "all"; + }; + + services.restic.server.prometheus = true; + + # miniflux + sops.templates."miniflux_metrics_env" = { + content = '' + METRICS_COLLECTOR=1 + LOG_LEVEL=debug + METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"} + METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"} + ''; + group = "prometheus-auth"; + mode = "0440"; + }; + + systemd.services.miniflux.serviceConfig = { + EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ]; + SupplementaryGroups = [ "prometheus-auth" ]; + }; + + services.ntfy-sh.settings.enable-metrics = true; + + services.caddy.globalConfig = '' + servers { + metrics + } + + admin ${config.networking.hostName}.coho-tet.ts.net:2019 { + } + ''; + }; +} diff --git a/modules/nixos/monitor/grafana.nix b/modules/nixos/monitor/grafana.nix new file mode 100644 index 0000000..e1b2cf3 --- /dev/null +++ b/modules/nixos/monitor/grafana.nix @@ -0,0 +1,43 @@ +{ config, lib, ... }: +let + cfg = config.custom.monitoring.grafana; +in +{ + config = lib.mkIf cfg.enable { + sops.templates."grafana.env".content = '' + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${config.sops.placeholder."grafana/oauth_secret"} + ''; + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3003; + root_url = "https://grafana.xinyang.life"; + domain = "grafana.xinyang.life"; + }; + "auth.generic_oauth" = { + enabled = true; + name = "Kanidm"; + client_id = "grafana"; + scopes = "openid,profile,email,groups"; + auth_url = "https://auth.xinyang.life/ui/oauth2"; + token_url = "https://auth.xinyang.life/oauth2/token"; + api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo"; + use_pkce = true; + use_refresh_token = true; + allow_sign_up = true; + login_attribute_path = "preferred_username"; + groups_attribute_path = "groups"; + role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'"; + allow_assign_grafana_admin = true; + auto_login = true; + }; + "auth" = { + disable_login_form = true; + }; + }; + }; + systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.templates."grafana.env".path; + }; +} diff --git a/modules/nixos/prometheus/restic.nix b/modules/nixos/monitor/restic.nix similarity index 71% rename from modules/nixos/prometheus/restic.nix rename to modules/nixos/monitor/restic.nix index a3ab710..24dcfa8 100644 --- a/modules/nixos/prometheus/restic.nix +++ b/modules/nixos/monitor/restic.nix @@ -3,17 +3,26 @@ let cfg = config.custom.prometheus; in { - config = lib.mkIf (cfg.enable && cfg.exporters.restic.enable) { + config = { services.restic.server.prometheus = true; - services.prometheus.scrapeConfigs = [ - (lib.mkIf cfg.exporters.restic.enable { - job_name = "restic"; - static_configs = [ { targets = [ config.services.restic.server.listenAddress ]; } ]; - }) - ]; + custom.prometheus.templates.scrape.mkResticScrapes = + { + address, + port ? null, + ... + }: + let + portStr = if port then ":${toString port}" else ""; + in + [ + (lib.mkIf cfg.exporters.restic.enable { + job_name = "restic"; + static_configs = [ { targets = [ "${address}${portStr}" ]; } ]; + }) + ]; - custom.prometheus.ruleModules = [ + custom.prometheus.templates.rules.mkResticRules = [ { name = "restic_alerts"; rules = [ diff --git a/modules/nixos/prometheus/blackbox.nix b/modules/nixos/prometheus/blackbox.nix deleted file mode 100644 index 1bfd896..0000000 --- a/modules/nixos/prometheus/blackbox.nix +++ /dev/null @@ -1,93 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.blackbox.enable) { - services.prometheus.exporters.blackbox = { - enable = true; - listenAddress = "127.0.0.1"; - configFile = pkgs.writeText "blackbox.config.yaml" ( - lib.generators.toYAML { } { - modules = { - tcp4_connect = { - prober = "tcp"; - tcp = { - ip_protocol_fallback = false; - preferred_ip_protocol = "ip4"; - tls = false; - }; - timeout = "15s"; - }; - }; - } - ); - }; - - services.prometheus.scrapeConfigs = [ - { - job_name = "blackbox"; - scrape_interval = "1m"; - metrics_path = "/probe"; - params = { - module = [ "tcp4_connect" ]; - }; - static_configs = [ - { - targets = [ - "tok-00.namely.icu:8080" - "la-00.video.namely.icu:8080" - "auth.xinyang.life:443" - "home.xinyang.life:8000" - ]; - } - ]; - relabel_configs = [ - { - source_labels = [ "__address__" ]; - target_label = "__param_target"; - } - { - source_labels = [ "__param_target" ]; - target_label = "instance"; - } - { - target_label = "__address__"; - replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; - } - ]; - } - { - job_name = "blackbox_exporter"; - static_configs = [ - { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; } - ]; - } - ]; - - custom.prometheus.ruleModules = [ - { - name = "probe_alerts"; - rules = [ - { - alert = "HighProbeLatency"; - expr = "probe_duration_seconds > 0.5"; - for = "2m"; - labels = { - severity = "warning"; - }; - annotations = { - summary = "High request latency on {{ $labels.instance }}"; - description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes."; - }; - } - ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/caddy.nix b/modules/nixos/prometheus/caddy.nix deleted file mode 100644 index 98e6783..0000000 --- a/modules/nixos/prometheus/caddy.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.caddy.enable) { - services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable '' - servers { - metrics - } - ''; - - services.prometheus.scrapeConfigs = [ - { - job_name = "caddy"; - static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ]; - } - ]; - - custom.prometheus.ruleModules = [ - { - name = "caddy_alerts"; - rules = [ - { - alert = "UpstreamHealthy"; - expr = "caddy_reverse_proxy_upstreams_healthy != 1"; - for = "5m"; - labels = { - severity = "critical"; - }; - annotations = { - summary = "Upstream {{ $labels.unstream }} not healthy"; - }; - } - ]; - } - ]; - }; - -} diff --git a/modules/nixos/prometheus/default.nix b/modules/nixos/prometheus/default.nix deleted file mode 100644 index ed2544a..0000000 --- a/modules/nixos/prometheus/default.nix +++ /dev/null @@ -1,230 +0,0 @@ -{ - config, - pkgs, - lib, - ... -}: - -with lib; - -let - cfg = config.custom.prometheus; - mkExporterOption = - enableOption: - (mkOption { - type = types.bool; - default = enableOption; - description = "Enable this exporter"; - }); - - mkRulesOption = mkOption { - type = types.listOf ( - types.submodule { - options = { - name = mkOption { type = lib.types.str; }; - rules = mkOption { type = lib.types.listOf lib.types.attrs; }; - }; - } - ); - }; -in -{ - imports = [ - ./blackbox.nix - ./caddy.nix - ./gotosocial.nix - ./immich.nix - ./miniflux.nix - ./ntfy-sh.nix - ./restic.nix - ]; - - options = { - custom.prometheus = { - enable = mkEnableOption "Prometheus instance"; - exporters = { - enable = mkOption { - type = types.bool; - default = false; - description = "Enable Prometheus exporter on every supported services"; - }; - - restic.enable = mkExporterOption config.services.restic.server.enable; - blackbox.enable = mkExporterOption false; - caddy.enable = mkExporterOption config.services.caddy.enable; - gotosocial.enable = mkExporterOption config.services.gotosocial.enable; - immich.enable = mkExporterOption config.services.immich.enable; - miniflux.enable = mkExporterOption config.services.miniflux.enable; - ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable; - }; - grafana = { - enable = mkEnableOption "Grafana Cloud"; - password_file = mkOption { type = types.path; }; - }; - ruleModules = mkRulesOption; - }; - }; - - config = mkIf cfg.enable { - services.tailscale = { - enable = true; - permitCertUid = config.services.caddy.user; - openFirewall = true; - }; - - services.caddy = { - enable = true; - virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = '' - reverse_proxy 127.0.0.1:${toString config.services.prometheus.port} - ''; - }; - - services.prometheus = mkIf cfg.enable { - enable = true; - port = 9091; - globalConfig.external_labels = { - hostname = config.networking.hostName; - }; - remoteWrite = mkIf cfg.grafana.enable [ - { - name = "grafana"; - url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push"; - basic_auth = { - username = "1340065"; - password_file = cfg.grafana.password_file; - }; - } - ]; - exporters = { - node = { - enable = true; - enabledCollectors = [ - "loadavg" - "time" - "systemd" - ]; - listenAddress = "127.0.0.1"; - port = 9100; - }; - }; - scrapeConfigs = [ - { - job_name = "prometheus"; - static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ]; - } - { - job_name = "node"; - static_configs = [ - { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } - ]; - } - ]; - - alertmanager = { - enable = true; - listenAddress = "127.0.0.1"; - logLevel = "debug"; - configuration = { - route = { - receiver = "ntfy"; - }; - receivers = [ - { - name = "ntfy"; - webhook_configs = [ - { - url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL '' - Alert {{.status}} - {{range .alerts}}-----{{range $k,$v := .labels}} - {{$k}}={{$v}}{{end}} - {{end}} - ''}"; - send_resolved = true; - } - ]; - } - ]; - }; - }; - - alertmanagers = [ - { - scheme = "http"; - static_configs = [ - { - targets = [ - "${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}" - ]; - } - ]; - } - ]; - rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ]; - }; - custom.prometheus.ruleModules = [ - { - name = "system_alerts"; - rules = [ - { - alert = "SystemdFailedUnits"; - expr = "node_systemd_unit_state{state=\"failed\"} > 0"; - for = "5m"; - labels = { - severity = "critical"; - }; - annotations = { - summary = "Systemd has failed units on {{ $labels.instance }}"; - description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; - }; - } - { - alert = "HighLoadAverage"; - expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})"; - for = "1m"; - labels = { - severity = "warning"; - }; - annotations = { - summary = "High load average detected on {{ $labels.instance }}"; - description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; - }; - } - { - alert = "HighTransmitTraffic"; - expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000"; - for = "1m"; - labels = { - severity = "warning"; - }; - annotations = { - summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; - description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; - }; - } - { - alert = "NetworkTrafficExceedLimit"; - expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200''; - for = "0m"; - labels = { - severity = "critical"; - }; - annotations = { - summary = "Outbound network traffic exceed 300GB for last 30 day"; - }; - } - { - alert = "JobDown"; - expr = "up == 0"; - for = "1m"; - labels = { - severity = "critical"; - }; - annotations = { - summary = "Job {{ $labels.job }} down for 1m."; - }; - } - ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/gotosocial.nix b/modules/nixos/prometheus/gotosocial.nix deleted file mode 100644 index e5da05e..0000000 --- a/modules/nixos/prometheus/gotosocial.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.gotosocial.enable) { - services.gotosocial.settings = { - metrics-enabled = true; - }; - services.prometheus.scrapeConfigs = [ - { - job_name = "gotosocial"; - static_configs = [ { targets = [ "localhost:8080" ]; } ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/immich.nix b/modules/nixos/prometheus/immich.nix deleted file mode 100644 index 4b92500..0000000 --- a/modules/nixos/prometheus/immich.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; - immichEnv = config.services.immich.environment; - metricPort = - if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv then - immichEnv.IMMICH_API_METRICS_PORT - else - 8081; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) { - services.immich.environment = { - IMMICH_METRICS = "true"; - }; - - services.prometheus.scrapeConfigs = [ - { - job_name = "immich"; - static_configs = [ { targets = [ "127.0.0.1:${toString metricPort}" ]; } ]; - } - ]; - }; - -} diff --git a/modules/nixos/prometheus/miniflux.nix b/modules/nixos/prometheus/miniflux.nix deleted file mode 100644 index b437b00..0000000 --- a/modules/nixos/prometheus/miniflux.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.miniflux.enable) { - systemd.services.miniflux.environment.METRICS_COLLECTOR = "1"; - services.prometheus.scrapeConfigs = [ - { - job_name = "miniflux"; - static_configs = [ { targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; } ]; - } - ]; - }; -} diff --git a/modules/nixos/prometheus/ntfy-sh.nix b/modules/nixos/prometheus/ntfy-sh.nix deleted file mode 100644 index 94e81f7..0000000 --- a/modules/nixos/prometheus/ntfy-sh.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ config, lib, ... }: -let - cfg = config.custom.prometheus; -in -{ - config = lib.mkIf (cfg.enable && cfg.exporters.ntfy-sh.enable) { - services.ntfy-sh.settings.enable-metrics = true; - services.prometheus.scrapeConfigs = [ - { - job_name = "ntfy-sh"; - static_configs = [ { targets = [ "ntfy.xinyang.life" ]; } ]; - } - ]; - }; -} diff --git a/overlays/my-lib/default.nix b/overlays/my-lib/default.nix new file mode 100644 index 0000000..8d07bc1 --- /dev/null +++ b/overlays/my-lib/default.nix @@ -0,0 +1,3 @@ +{ +} +// (import ./prometheus.nix) diff --git a/overlays/my-lib/prometheus.nix b/overlays/my-lib/prometheus.nix new file mode 100644 index 0000000..da43f77 --- /dev/null +++ b/overlays/my-lib/prometheus.nix @@ -0,0 +1,209 @@ +let + mkFunction = f: (targets: (map f targets)); + mkPort = port: if isNull port then "" else ":${toString port}"; +in +{ + mkScrapes = mkFunction ( + { + name, + address, + passwordFile ? null, + port ? 443, + scheme ? "https", + ... + }: + { + job_name = "${name}(${address})"; + scheme = scheme; + static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ]; + } + // ( + if isNull passwordFile then + { } + else + { + basic_auth.username = "prom"; + basic_auth.password_file = passwordFile; + } + ) + ); + + mkCaddyScrapes = mkFunction ( + { + address, + port ? 2019, + ... + }: + { + job_name = "caddy_${address}"; + static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ]; + } + ); + + mkCaddyRules = mkFunction ( + { + host ? "", + name ? "caddy_alerts_${host}", + }: + { + inherit name; + rules = [ + { + alert = "UpstreamHealthy"; + expr = "caddy_reverse_proxy_upstreams_healthy != 1"; + for = "5m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "Upstream {{ $labels.unstream }} not healthy"; + }; + } + ]; + } + ); + + mkNodeScrapes = mkFunction ( + { + address, + port ? 9100, + ... + }: + { + job_name = "node_${address}"; + static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ]; + } + ); + + mkNodeRules = mkFunction ( + { + host ? "", + name ? "system_alerts_${host}", + ... + }: + { + inherit name; + rules = [ + { + alert = "SystemdFailedUnits"; + expr = "node_systemd_unit_state{state=\"failed\"} > 0"; + for = "5m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "Systemd has failed units on {{ $labels.instance }}"; + description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!"; + }; + } + { + alert = "HighLoadAverage"; + expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})"; + for = "1m"; + labels = { + severity = "warning"; + }; + annotations = { + summary = "High load average detected on {{ $labels.instance }}"; + description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs."; + }; + } + { + alert = "HighTransmitTraffic"; + expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000"; + for = "1m"; + labels = { + severity = "warning"; + }; + annotations = { + summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})"; + description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute."; + }; + } + { + alert = "NetworkTrafficExceedLimit"; + expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200''; + for = "0m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "Outbound network traffic exceed 300GB for last 30 day"; + }; + } + ]; + } + ); + + mkBlackboxScrapes = mkFunction ( + { + hostAddress, + hostPort ? 9115, + targetAddresses, + ... + }: + { + job_name = "blackbox(${hostAddress})"; + scrape_interval = "1m"; + metrics_path = "/probe"; + params = { + module = [ "tcp4_connect" ]; + }; + static_configs = [ + { + targets = targetAddresses; + } + ]; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "${hostAddress}${mkPort hostPort}"; + } + ]; + } + ); + + mkBlackboxRules = mkFunction ( + { + host ? "", + name ? "probe_alerts_${host}", + }: + { + inherit name; + rules = [ + { + alert = "HighProbeLatency"; + expr = "probe_duration_seconds > 0.5"; + for = "3m"; + labels = { + severity = "warning"; + }; + annotations = { + summary = "High request latency on {{ $labels.instance }}"; + description = "Request latency is above 0.5 seconds for the last 3 minutes."; + }; + } + { + alert = "VeryHighProbeLatency"; + expr = "probe_duration_seconds > 1"; + for = "3m"; + labels = { + severity = "critical"; + }; + annotations = { + summary = "High request latency on {{ $labels.instance }}"; + description = "Request latency is above 0.5 seconds for the last 3 minutes."; + }; + } + ]; + } + ); +}