Compare commits
5 commits
ebf69d94dd
...
92db38383e
| Author | SHA1 | Date | |
|---|---|---|---|
92db38383e
|
|||
|
74b67e1854
|
|||
|
97fcdefc2b
|
|||
|
9e3af9a535
|
|||
|
ec63c8b6cc
|
32 changed files with 833 additions and 639 deletions
37
.sops.yaml
37
.sops.yaml
|
|
@ -2,13 +2,22 @@ keys:
|
||||||
- &xin age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
|
- &xin age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
|
||||||
- &host-calcite age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
|
- &host-calcite age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
|
||||||
- &host-raspite age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
|
- &host-raspite age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
|
||||||
- &host-sgp-00 age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
|
|
||||||
- &host-la-00 age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
|
- &host-la-00 age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
|
||||||
- &host-massicot age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
|
- &host-massicot age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
|
||||||
- &host-weilite age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml
|
- &host-weilite age17r3fxfmt6hgwe984w4lds9u0cnkf5ttq8hnqt800ayfmx7t8t5gqjddyml
|
||||||
- &host-hk-00 age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
|
- &host-hk-00 age1p2dlc8gfgyrvtta6mty2pezjycn244gmvh456qd3wvkfwesp253qnwyta9
|
||||||
- &host-fra-00 age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
|
- &host-fra-00 age18u4mqrhqkrpcytxfxfex6aeap04u38emhy6u4wrp5k62sz2vae4qm5jj7s
|
||||||
|
- &host-biotite age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
|
||||||
|
- &host-thorite age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
|
||||||
creation_rules:
|
creation_rules:
|
||||||
|
- path_regex: machines/secrets.yaml
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *xin
|
||||||
|
- *host-calcite
|
||||||
|
- *host-massicot
|
||||||
|
- *host-thorite
|
||||||
|
- *host-biotite
|
||||||
- path_regex: machines/calcite/secrets.yaml
|
- path_regex: machines/calcite/secrets.yaml
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|
@ -24,19 +33,23 @@ creation_rules:
|
||||||
- age:
|
- age:
|
||||||
- *xin
|
- *xin
|
||||||
- *host-massicot
|
- *host-massicot
|
||||||
|
- paht_regex: machines/biotite/secrets.yaml
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *xin
|
||||||
|
- *host-biotite
|
||||||
|
- path_regex: machines/thorite/secrets.yaml
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *xin
|
||||||
|
- *host-thorite
|
||||||
- path_regex: machines/dolomite/secrets/secrets.yaml
|
- path_regex: machines/dolomite/secrets/secrets.yaml
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
- *xin
|
- *xin
|
||||||
- *host-sgp-00
|
|
||||||
- *host-la-00
|
- *host-la-00
|
||||||
- *host-hk-00
|
- *host-hk-00
|
||||||
- *host-fra-00
|
- *host-fra-00
|
||||||
- path_regex: machines/dolomite/secrets/sgp-00.yaml
|
|
||||||
key_groups:
|
|
||||||
- age:
|
|
||||||
- *xin
|
|
||||||
- *host-sgp-00
|
|
||||||
- path_regex: machines/dolomite/secrets/la-00.yaml
|
- path_regex: machines/dolomite/secrets/la-00.yaml
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|
@ -58,16 +71,6 @@ creation_rules:
|
||||||
- age:
|
- age:
|
||||||
- *xin
|
- *xin
|
||||||
- *host-weilite
|
- *host-weilite
|
||||||
- path_regex: machines/secrets.yaml
|
|
||||||
key_groups:
|
|
||||||
- age:
|
|
||||||
- *xin
|
|
||||||
- *host-calcite
|
|
||||||
- *host-raspite
|
|
||||||
- *host-sgp-00
|
|
||||||
- *host-la-00
|
|
||||||
- *host-hk-00
|
|
||||||
- *host-massicot
|
|
||||||
- path_regex: home/xin/secrets.yaml
|
- path_regex: home/xin/secrets.yaml
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|
|
||||||
27
flake.nix
27
flake.nix
|
|
@ -60,8 +60,8 @@
|
||||||
outputs =
|
outputs =
|
||||||
{
|
{
|
||||||
self,
|
self,
|
||||||
home-manager,
|
|
||||||
nixpkgs,
|
nixpkgs,
|
||||||
|
home-manager,
|
||||||
nixos-hardware,
|
nixos-hardware,
|
||||||
sops-nix,
|
sops-nix,
|
||||||
flake-utils,
|
flake-utils,
|
||||||
|
|
@ -84,6 +84,7 @@
|
||||||
overlayModule =
|
overlayModule =
|
||||||
{ ... }:
|
{ ... }:
|
||||||
{
|
{
|
||||||
|
_module.args.my-lib = import ./overlays/my-lib;
|
||||||
nixpkgs.overlays = [
|
nixpkgs.overlays = [
|
||||||
editorOverlay
|
editorOverlay
|
||||||
(import ./overlays/add-pkgs.nix)
|
(import ./overlays/add-pkgs.nix)
|
||||||
|
|
@ -118,10 +119,6 @@
|
||||||
./machines/dolomite/bandwagon.nix
|
./machines/dolomite/bandwagon.nix
|
||||||
./machines/dolomite/common.nix
|
./machines/dolomite/common.nix
|
||||||
];
|
];
|
||||||
tok-00 = [
|
|
||||||
./machines/dolomite/lightsail.nix
|
|
||||||
./machines/dolomite/common.nix
|
|
||||||
];
|
|
||||||
fra-00 = [
|
fra-00 = [
|
||||||
./machines/dolomite/fra.nix
|
./machines/dolomite/fra.nix
|
||||||
./machines/dolomite/common.nix
|
./machines/dolomite/common.nix
|
||||||
|
|
@ -178,7 +175,6 @@
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
nixpkgs = nixpkgs;
|
|
||||||
nixosModules.default = {
|
nixosModules.default = {
|
||||||
imports = [
|
imports = [
|
||||||
./modules/nixos
|
./modules/nixos
|
||||||
|
|
@ -190,7 +186,9 @@
|
||||||
colmenaHive = colmena.lib.makeHive {
|
colmenaHive = colmena.lib.makeHive {
|
||||||
meta = {
|
meta = {
|
||||||
# FIXME:
|
# FIXME:
|
||||||
nixpkgs = import nixpkgs { system = "x86_64-linux"; };
|
nixpkgs = import nixpkgs {
|
||||||
|
system = "x86_64-linux";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
massicot =
|
massicot =
|
||||||
|
|
@ -205,20 +203,6 @@
|
||||||
] ++ sharedColmenaModules;
|
] ++ sharedColmenaModules;
|
||||||
};
|
};
|
||||||
|
|
||||||
tok-00 =
|
|
||||||
{ ... }:
|
|
||||||
{
|
|
||||||
imports = nodeNixosModules.tok-00 ++ sharedColmenaModules;
|
|
||||||
nixpkgs.system = "x86_64-linux";
|
|
||||||
networking.hostName = "tok-00";
|
|
||||||
system.stateVersion = "23.11";
|
|
||||||
deployment = {
|
|
||||||
targetHost = "video01.namely.icu";
|
|
||||||
buildOnTarget = false;
|
|
||||||
tags = [ "proxy" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
la-00 =
|
la-00 =
|
||||||
{ ... }:
|
{ ... }:
|
||||||
{
|
{
|
||||||
|
|
@ -310,7 +294,6 @@
|
||||||
osmium = mkNixos {
|
osmium = mkNixos {
|
||||||
hostname = "osmium";
|
hostname = "osmium";
|
||||||
};
|
};
|
||||||
|
|
||||||
} // self.colmenaHive.nodes;
|
} // self.colmenaHive.nodes;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,11 +12,12 @@
|
||||||
networking.useNetworkd = true;
|
networking.useNetworkd = true;
|
||||||
systemd.network.enable = true;
|
systemd.network.enable = true;
|
||||||
systemd.network.networks."10-wan" = {
|
systemd.network.networks."10-wan" = {
|
||||||
matchConfig.MACAddress = "00:16:3e:0a:ec:45";
|
matchConfig.MACAddress = "b6:20:0d:9a:6c:34";
|
||||||
networkConfig.DHCP = "ipv4";
|
networkConfig = {
|
||||||
dhcpV4Config = {
|
DHCP = "ipv4";
|
||||||
UseDNS = true;
|
IPv6SendRA = true;
|
||||||
};
|
};
|
||||||
|
address = [ "2a03:4000:4a:148::1/64" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
commonSettings = {
|
commonSettings = {
|
||||||
|
|
|
||||||
|
|
@ -368,8 +368,7 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
custom.prometheus = {
|
custom.prometheus = {
|
||||||
enable = true;
|
exporters.node.enable = true;
|
||||||
exporters.blackbox.enable = true;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.ollama = {
|
services.ollama = {
|
||||||
|
|
|
||||||
|
|
@ -31,14 +31,7 @@
|
||||||
fsType = "ext4";
|
fsType = "ext4";
|
||||||
};
|
};
|
||||||
|
|
||||||
swapDevices = [ ];
|
networking.useNetworkd = true;
|
||||||
|
|
||||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
|
||||||
# (the default) this is the recommended approach. When using systemd-networkd it's
|
|
||||||
# still possible to use this option, but it's recommended to use it in conjunction
|
|
||||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
|
||||||
# networking.useNetworkd = false;
|
|
||||||
|
|
||||||
systemd.network.enable = true;
|
systemd.network.enable = true;
|
||||||
systemd.network.networks."10-wan" = {
|
systemd.network.networks."10-wan" = {
|
||||||
matchConfig.MACAddress = "00:16:3e:0a:ec:45";
|
matchConfig.MACAddress = "00:16:3e:0a:ec:45";
|
||||||
|
|
@ -47,7 +40,6 @@
|
||||||
UseDNS = true;
|
UseDNS = true;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
# networking.interfaces.eth0.useDHCP = lib.mkDefault true;
|
|
||||||
|
|
||||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -21,12 +21,20 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
swapDevices = [
|
||||||
|
{
|
||||||
|
device = "/swapfile";
|
||||||
|
size = 2 * 1024;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
custom.prometheus = {
|
custom.prometheus.exporters = {
|
||||||
enable = lib.mkDefault true;
|
enable = true;
|
||||||
exporters.blackbox.enable = true;
|
node.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.tailscale.enable = true;
|
||||||
|
|
||||||
commonSettings = {
|
commonSettings = {
|
||||||
auth.enable = true;
|
auth.enable = true;
|
||||||
proxyServer = {
|
proxyServer = {
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,12 @@
|
||||||
networkConfig = {
|
networkConfig = {
|
||||||
DHCP = "no";
|
DHCP = "no";
|
||||||
Gateway = "185.217.108.1";
|
Gateway = "185.217.108.1";
|
||||||
|
DNSSEC = true;
|
||||||
|
DNSOverTLS = true;
|
||||||
|
DNS = [
|
||||||
|
"8.8.8.8#dns.google"
|
||||||
|
"8.8.4.4#dns.google"
|
||||||
|
];
|
||||||
};
|
};
|
||||||
address = [ "185.217.108.59/24" ];
|
address = [ "185.217.108.59/24" ];
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -24,10 +24,6 @@
|
||||||
hedgedoc_env = {
|
hedgedoc_env = {
|
||||||
owner = "hedgedoc";
|
owner = "hedgedoc";
|
||||||
};
|
};
|
||||||
grafana_cloud_api = {
|
|
||||||
owner = "prometheus";
|
|
||||||
sopsFile = ../secrets.yaml;
|
|
||||||
};
|
|
||||||
grafana_oauth_secret = {
|
grafana_oauth_secret = {
|
||||||
owner = "grafana";
|
owner = "grafana";
|
||||||
};
|
};
|
||||||
|
|
@ -62,6 +58,8 @@
|
||||||
hostName = "massicot";
|
hostName = "massicot";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.tailscale.enable = true;
|
||||||
|
|
||||||
commonSettings = {
|
commonSettings = {
|
||||||
auth.enable = true;
|
auth.enable = true;
|
||||||
nix = {
|
nix = {
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,20 @@
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
gotosocial = {
|
||||||
|
displayName = "GoToSocial";
|
||||||
|
originUrl = "https://gts.xiny.li/auth/callback";
|
||||||
|
originLanding = "https://gts.xiny.li/auth/callback";
|
||||||
|
allowInsecureClientDisablePkce = true;
|
||||||
|
scopeMaps = {
|
||||||
|
gts-users = [
|
||||||
|
"openid"
|
||||||
|
"email"
|
||||||
|
"profile"
|
||||||
|
"groups"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
# It's used for all the clients. I'm too lazy to change the name.
|
# It's used for all the clients. I'm too lazy to change the name.
|
||||||
owncloud-android = {
|
owncloud-android = {
|
||||||
displayName = "ownCloud Apps";
|
displayName = "ownCloud Apps";
|
||||||
|
|
@ -147,7 +161,7 @@
|
||||||
immich = {
|
immich = {
|
||||||
displayName = "Immich";
|
displayName = "Immich";
|
||||||
originUrl = [
|
originUrl = [
|
||||||
"https://immich.xinyang.life:8000/api/oauth/mobile-redirect"
|
"https://immich.xinyang.life:8000/api/oauth/mobile-redirect/"
|
||||||
"https://immich.xinyang.life:8000/auth/login"
|
"https://immich.xinyang.life:8000/auth/login"
|
||||||
"https://immich.xinyang.life:8000/user-settings"
|
"https://immich.xinyang.life:8000/user-settings"
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,10 @@
|
||||||
networking.useNetworkd = true;
|
networking.useNetworkd = true;
|
||||||
systemd.network.networks."10-wan" = {
|
systemd.network.networks."10-wan" = {
|
||||||
matchConfig.MACAddress = "96:00:02:68:7d:2d";
|
matchConfig.MACAddress = "96:00:02:68:7d:2d";
|
||||||
networkConfig.DHCP = "ipv4";
|
networkConfig = {
|
||||||
networkConfig.Gateway = "fe80::1";
|
DHCP = "ipv4";
|
||||||
|
Gateway = "fe80::1";
|
||||||
|
};
|
||||||
address = [
|
address = [
|
||||||
"2a01:4f8:c17:345f::3/64"
|
"2a01:4f8:c17:345f::3/64"
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -43,10 +43,14 @@ in
|
||||||
environmentFile = config.sops.secrets.hedgedoc_env.path;
|
environmentFile = config.sops.secrets.hedgedoc_env.path;
|
||||||
};
|
};
|
||||||
|
|
||||||
custom.prometheus = {
|
custom.prometheus.exporters = {
|
||||||
enable = true;
|
enable = true;
|
||||||
exporters.blackbox.enable = true;
|
blackbox = {
|
||||||
exporters.miniflux.enable = true;
|
enable = true;
|
||||||
|
};
|
||||||
|
node = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
security.acme = {
|
security.acme = {
|
||||||
|
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
{ pkgs, config, ... }:
|
|
||||||
{
|
|
||||||
sops.secrets = {
|
|
||||||
autofs-nas = {
|
|
||||||
owner = "davfs2";
|
|
||||||
};
|
|
||||||
autofs-nas-secret = {
|
|
||||||
path = "/etc/davfs2/secrets";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
fileSystems."/media/nas" = {
|
|
||||||
device = "https://home.xinyang.life:5244/dav";
|
|
||||||
fsType = "davfs";
|
|
||||||
options = [
|
|
||||||
"uid=1000"
|
|
||||||
"gid=1000"
|
|
||||||
"rw"
|
|
||||||
"_netdev"
|
|
||||||
];
|
|
||||||
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,14 +1,6 @@
|
||||||
clash_subscription_link: ENC[AES256_GCM,data:uDaX2BE/qRdfXVtckX0VKpu0LN3j0YaxVIPbQt3tGAfdfqFqlp0IzFgNiZBIEcIltYkeEyqFSA0QnttoMb0QYe9f2rtgjztwk10SOGViGaeFWPfkdlHP04qhm5OOOddi3OwT5rUNwvBU79AdCnLJ9QwqMbOaNm/JTtbkcjf8huxc2UcYAQcY/YNJ7aTEhWIw98Ab85aih+w=,iv:pZ189IPPCBjscXzEdgQCRdFlls3TniwDfNCd+H1FFaQ=,tag:dMmGZvppWtkc82b5dTnJwg==,type:str]
|
prometheus:
|
||||||
autofs-nas: ENC[AES256_GCM,data:LnCKGKARx6Vd99VwAX/6PXOJwo+a7GP8fNmM9yuuC2xITGxtWCsDdOZL1+IA5LS/gbOYINgQWDzWirJF3LCP27BQeLwXYpD7/UAwwVI=,iv:QJzsS5a6vWeoBxkB13yXdVbyn0tt2QTvqj0LaHn6S2g=,tag:D/JKXQIw1EzIh3wjGhHgHg==,type:str]
|
metrics_username: ENC[AES256_GCM,data:/CQfOA==,iv:BjhB+uLfjmYHdgpc/+tDJXJ8C1EK9kngQWbo4NleOmE=,tag:JCdqyqGLRh09T25vmufiZw==,type:str]
|
||||||
autofs-nas-secret: ENC[AES256_GCM,data:gbOizRZAvh79HlJWIWeKTk79Ux311XGL1eIswc0P2U2huCibD/ji3kOlSjZXENG+fJQKNz2AlDTk3g2cQQ==,iv:UCaGeE8j4RqJzA0xhu3oB2xvzombzQD3fjLKCWd5fDg=,tag:II4eEMr7f2TDUl1qUcDYXA==,type:str]
|
metrics_password: ENC[AES256_GCM,data:q/xMPuNtlcUFewMdVu6w2Q==,iv:xLohdb5tdxevYFckZoacjSJp2rZ53QKLxK6u3mc3mDw=,tag:B4LrObH1DsnnD5CcuOPOyg==,type:str]
|
||||||
github_public_token: ENC[AES256_GCM,data:6Gt+oJcCRHeoLK7CRndMMbszTXSEbnN0nQzsVOnl/+zB4hxbEPD5k/vkkl+cZ/qmxdxFXV0OOsYvktn44Yv1DMUE3mkB0hcAdoyPwLuYM7W3RpOoW3OktH8DRCUi6msvFp3ykpdmIl9WyjVhc/lMwTaYJQyRh1ue,iv:PJSFtJBelyc3rzd6hqjMp+ciU2Q3FTOEXsiq5F2KKTY=,tag:Y/stRg6kwyjjIFZCXS/peg==,type:str]
|
|
||||||
singbox_sg_server: ENC[AES256_GCM,data:SF2ja6W4TwThwoug5x2KTA==,iv:Vx9wNTdVHkReux4YeQY+0VkC1Wqg/CRkY7frVY/3e50=,tag:7XA9KSoR0GA6FoYRhCv4BQ==,type:str]
|
|
||||||
singbox_jp_server: ENC[AES256_GCM,data:S3Bs5yVMzyz6vD51GYElOM5h,iv:nXetY339YuOi2jFEb3xkPTglHRMk/quIrQL4ko+8MxY=,tag:o9d55cZuWmX4NDYexWjvYQ==,type:str]
|
|
||||||
sing-box:
|
|
||||||
password: ENC[AES256_GCM,data:xyqmoJEDI5959zHPTVelln/iThtoeDwS,iv:rLyqJsE/4JDf08RlMLLPh+MKJkba9bL0z8jx6bTEfgc=,tag:cgLHdeLIyPvLhRNaVcQ0TQ==,type:str]
|
|
||||||
uuid: ENC[AES256_GCM,data:lWBCM5wyz6BcUUHdvynkn5y166Kk15jO0EhWUDuhXXhrve5l,iv:RmDJYFnYqIEIShLn25sf4h8AO2E3+3Xa2U9Mff+Xk2w=,tag:SN0DUdwZXKO/VEnozrr5mA==,type:str]
|
|
||||||
grafana_cloud_api: ENC[AES256_GCM,data:eEvPAwtThK1FMhbrnmSo89+GlWZAF+LQRMLXA2C6f1vR7ZPlXJZGWzjYwDcPlnpiC737/cG14M4kZqvPGBuNub5A83rBS/+FeebvGDIF59L5PC1Ys1jWBB9YRI/L9EU0tvwTTUCvLRA9j28n7Jw7wR6mWXm63XA+OMu8/UbTwbeV/WUQn8vnwqadSUdCnNKJXMsAY+q9t/st0DPm5+aNxA==,iv:cHvbeCmLFmJPNKsl1BBYx9WJP7ZJWi+8c9yHZWc6FTs=,tag:87C+0FVvzDIowE0+QpY1zA==,type:str]
|
|
||||||
private_dns_address: ENC[AES256_GCM,data:YJxNOH4hsZHResvANEqJRTANhnL4PLp/Pmi/PhgtSTbTKiJKPqudhTEkNg==,iv:8+qG5rQXAKfrykEjt9qrbtyNaBuKvi7EaIWouRqEipY=,tag:VH0w5ZbXcWFGZ9GLavm7/w==,type:str]
|
|
||||||
sops:
|
sops:
|
||||||
kms: []
|
kms: []
|
||||||
gcp_kms: []
|
gcp_kms: []
|
||||||
|
|
@ -18,77 +10,50 @@ sops:
|
||||||
- recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
|
- recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiRzZVNGFocUN2VzZLTmJz
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmUUtKcU0va1F3SjFzcktl
|
||||||
WlJnUmxhZS92citDRkVZVnJZQU9YWVZORlNjCkgzeWl5dTl1YmpjZGt2anF3dGgr
|
S0FJNHlTUzBkMWlydGRjM1Y4SGd2SXpMVTJRCmtRSW9wNW9xMDBaQ0YzUWM4YjRz
|
||||||
K1hOSTRmakNrZ2JoNit3NDIzK1FCcWMKLS0tIEdqY3VvR3gxd1JoQlhPR3JvcXBF
|
SVRDNHRjNG5hTHBOOHorTTlJU1BwY1EKLS0tIEpLREJ1VzFaalczZlhKaitHVTJU
|
||||||
K2g4VFpqUEF5RTQ3cmpUSG0xajN2bUUKMuwx5cO1nHokV1NOloXfl9wTBN/+/Rlq
|
MDdJaVBtVmw4WTlBUEF5WXJSVFRFeDAKnvF6CmnU8hxXSdKQPUJqPT7Dewl4REOH
|
||||||
UJKP/qaI23tpyMXN1U40iF20ecO1U5Ad8wAQ61C/tldSVULizDihpw==
|
wDQELRaDkMPMKEOAc6wCmXNErvj/I7w7wuvB5WxtanC7g4IEphD6aA==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
- recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
|
- recipient: age1ytwfqfeez3dqtazyjltn7mznccwx3ua8djhned7n8mxqhw4p6e5s97skfa
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYXlwdytVSm1SQzRubHdX
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBS2xqbWp0NkFMVm1Cci82
|
||||||
dHhrTWxyamo3OFRraEVRQ3plK1cwUWt0a1JRCkdqaVRTQ0NaTkdoMlpDT3Yyallq
|
d0ZmRXdRc2FFSklHNnZtV1Z0REFIelZDQ1U4CllmdllNVnp3WmpCeDNBRzVFbVR5
|
||||||
eTd0bDViVTgwZGRTUmlYTzR0Y09iWWcKLS0tIEFlQnFPVFRVNlAxdExMekJ2b295
|
WkJFMGs5ZWJEK1lSWDQwYUdOdFJseGcKLS0tIEZUU254aWtYdWthL3I2UkJ0eklj
|
||||||
UUJkUUZCNUZnbkNFZHVBYXNHQklOL2MKujgh6REuAKu6ZLVA7atiWUqhnvYJnQjb
|
WHhrRlRvLzlmY0REYktGSlh3MENzRzgKzO1XqXhcXAxfn86+IY+ccBII1SGYctAk
|
||||||
WsxCa9ZXZRgfbhcNlZ3qIKJpWWI/RMS17+Nm5yIl+2cSqe2UJMjZdg==
|
+ArpGmXaf53RFmPLSzMGNaiJzfhqk9U9bn3WV9CFdaA7Rtec0ZAcNw==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
|
||||||
- recipient: age1nugzw24upk8pz5lyz2z89qk8se4gpcsg3ypcs58nykncr56sevrsm8qpvj
|
|
||||||
enc: |
|
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNXQ4RVVRd3RYRkhUVExG
|
|
||||||
SHJON0hwSmJtUkYrd1dldHJRN2tPKzBsNlNjCi9xYVhaanF5TDU2Q0xadXNWR2tN
|
|
||||||
dHhQVkpRREFlRm5MM2pwVytEaGhHT0EKLS0tIE9sRUtLako3cnAxNm82RDhiWEVM
|
|
||||||
ZW1IMXkzYkhqbW1ZdVRabUlkK2oxSTgKHC22uQqMq+cJ7vrONkGgoH8snxGef6Ft
|
|
||||||
QbtoJziERjAhK6B7TOY8AJ3WVRpCzZN70HjLNYa+bMMNOvmlsVxfZg==
|
|
||||||
-----END AGE ENCRYPTED FILE-----
|
|
||||||
- recipient: age13s6rwd3wjk2x5wkn69tdczhl3l5d7mfmlv90efsv4q67jne43qss9tcakx
|
|
||||||
enc: |
|
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSmZ5YUpFdzRNdWZVNmxJ
|
|
||||||
bm5ucUhVeTV2TkE4ZElkZ1N3aXc2eVEwMlRNCmxXRElPb2pGYzJFVnUxQkRtMlNF
|
|
||||||
cjgwUzh5UWNLTk01U0h2bHNpaXVzZkkKLS0tIDczUkkwTG8rL1V3UU9lenk4V2tl
|
|
||||||
TUxDd2huTllMRG9MZTJZdzRwaWxqUVEKLA3y+heUA8cK31LZzv5A1wtgf+sauuwE
|
|
||||||
7SGU3uYU650tJM3e6Lveo+JOAD7Z1jrAomT5Bub+jjSHnpeFC9yMbA==
|
|
||||||
-----END AGE ENCRYPTED FILE-----
|
|
||||||
- recipient: age1t5nw2jx4dw67jkf72uxcxt72j7lq3xyj35lvl09f8kala90h2g2s2a5yvj
|
|
||||||
enc: |
|
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4enA2bnlrV3ViY1RHaTVS
|
|
||||||
Ym5VV005NFlXZUl2NDNXYXBoOHh0SGQ0YVNnCm1KdHBSeE9lQzZEM2hFZUwzRitS
|
|
||||||
K3BEWGhtWmxKc2RJd0FTMEs2b1ArOUkKLS0tIG5kaWc3U0o4SG9teXk5dVZWWjkz
|
|
||||||
cS9VMU5YbEl3UE5mODJ1THNLVEdVblEKNQF0b9r1XPD819Z6Uy0b9hT4Uek2tNWU
|
|
||||||
3z3H7V/UiB1TMW+qgs6BC6bDkDf7oG//qmZEdYF+lDXcNSwai25xyg==
|
|
||||||
-----END AGE ENCRYPTED FILE-----
|
|
||||||
- recipient: age1fw2sqaa5s9c8ml6ncsexkj8ar4288387ju92ytjys4awf9aw6smqqz94dh
|
|
||||||
enc: |
|
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBreHJNRXlpOEh6YUxaSmJj
|
|
||||||
ZDlVdHh2b1p6aEs3eDAvbkk4WExxWmE5bDM0CkZzT2l1K09UbmNFNEpZUVY2NVlB
|
|
||||||
dVFYbnpvTjlUcTdZejMrelpscXRJQzAKLS0tIEVIaVByVmp0aUU1ZWJLajBhcjRk
|
|
||||||
QVZMRXBRVVhaY3JKZEJjMTdEeEVqcWMKT+DoevNQAxCrty2VkRDLWGFzs9GsW3F7
|
|
||||||
txz73tAceAIiocC1z7IV2TaYULYf7Z75HAje/SOTlGHBIDiVZ0vyLw==
|
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
- recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
|
- recipient: age1jle2auermhswqtehww9gqada8car5aczrx43ztzqf9wtcld0sfmqzaecta
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEZEJHQlJqMWxob1lxOUFK
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycmppSGR1YTMzTTJQSmx0
|
||||||
dDZIN3FaNWR1L0gyN3I1MXVXZlpzdlpQUHpjCmIwTWhRamZvSTF3cHZMNk9YUlRv
|
eFlRQWpPRTR1L042MXlqd3dURStjaUZjR1M4Ci9VdzZkSmN3d1BDNTlJdXNSenhZ
|
||||||
U2tOK3E5MFBFNERsUHVzVnhsUDFRd00KLS0tIFd1MUpaaFU0bWdVRjJ6NjFwcFZt
|
MDE1VktBN252L2FYMjJmNEFITGptM3MKLS0tIDNYOEZqNjM1VCtEWDlGdzYveG5j
|
||||||
bkJGWFFWanFBK1drZlBNcHo0c3Bjc00K/vPBLocRhtcJ3snGYFr+H7qhbg6iSSPP
|
dEp2bmVQMmV5ZU9Jb3FFSDFoT2NJOE0Kx1ZifyU2WLoHeUmqP9oCUmIl6ZJeytGB
|
||||||
OSH8WnaM5JmmA9IQlm5uGiG74PHi5sg5d+bwG8pPQtMKN+Ndxh7JIA==
|
WPMJKcNtuJHL1OWhT0wMiv6NEF5UaYXIlCqSVtXAMy554G4JlX5tQw==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
- recipient: age1w3x5mz2g8jc9aq8cajdpg62f8n5p4qr6jgjlxw9seagyw0t0fsuqvkmym0
|
- recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlN291MzZOaU4zazhEeXBh
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5OGlRWVM0Z01pYjdqYjEy
|
||||||
WlhoYmh2ZDBsZmc3cEthdW5paWpXbXQvUG1FCjBLZ0FPVWR3T2pVWTZrRmkxSWUr
|
TGdhNFhpNEUwSFZaeHNwRkpraFF5RTU2SXhnCm5YWmM5SmdERzZBWTgva1E4MDFm
|
||||||
MHhkUFFPK1Z0b2t1Z1J0VjlER1JvcGMKLS0tIE45YndxVW4vak1wcEJoZzhHQ0E0
|
N0xyUExGV0MvbFF3M0ZRSVEydFNUSGMKLS0tIGxxNWhsSEt4WDR2a2hId1JkVFE3
|
||||||
NzA1cy80ZW5vUFplQzVMZ0txSmVkMUEKFUvgmJNdo9sV33gOx7LVUSCYvIqCNwaP
|
enJ6MzJxR0I4eStSQk9ON0dsdjFmRkEKBSGkv1O0vgHSsU3+6AGN7bKQ5lpN7AMT
|
||||||
u+XoWTfg4kp9f4KVTy/8huPsVLhZBUaf6jI10mV2z4QwaLHje4JiHw==
|
eqEgWx7juZ7hKzLq1HMbiT61l0FrJNHEMfn15bzn7GsK5YJQvfiq9w==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
lastmodified: "2024-11-22T05:48:59Z"
|
- recipient: age1v5h946jfke6ae8pcgz52mhj26cacqcpl9dmmrrkf37x55rnq2v3szqctvv
|
||||||
mac: ENC[AES256_GCM,data:In/gSIYnXKbbv1lzS/nmSESCHBcBv/TtkvhzdNiIn73N4kP9aJ+1JE8Npix8zNItzk46DX+nHBk8Kwgl6uq26YtL+sMTBKh5K8Ny0H8ivlgS+olXswv3Y9h1cYD7FBHUKzbMuiJd0ppjC0ZIn20rRpb4d57rwUbvY0KstyQW4JA=,iv:DcdTAimbXXpKhhiB9rriS75+XGNOCcScqi/804+Xx6g=,tag:NHW+UViRmbUDHb0gTd9TDg==,type:str]
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjeVlxRk1PZUpUVm1xckRu
|
||||||
|
ZmMzdHdvKzExaTQxVHYvYjIvblo3b1ZORFJrCkpVdHFLbCtNS0xnamJ0T250YzUy
|
||||||
|
Uy9Xd0tMa3FSVlRkQXFaTWJVem9uWGsKLS0tIFRmT0VzL0hlLzkrRTZxcWtLN3Qv
|
||||||
|
YVMya3dUazFyaWRNNDJ3OVNIVXJLVTQK+7MxkmBjPszozXUO+zVaWdsovDmhWAfz
|
||||||
|
8puIpXpWZY09BkS0vs4oNhiVA9PD11TBIVCEbC5E1TwpwboMXBYhCQ==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
lastmodified: "2024-11-30T06:31:42Z"
|
||||||
|
mac: ENC[AES256_GCM,data:xh8x9IrQ01ZzdcCTIfBrifIGduMYVmSSP52BkTyr/bx7AgQAz2WeA7LFrccxIayCGHrQKfMQDLUKJ/EBamG/6p8AX6QqZBTfqFD688ZhmRfxgpj7fYR9jPYnhb/9XHI9R2jTaJWwrorXvu3pa+Gy/hWB3Kb+WZc3fslmIuKuLH0=,iv:GDrHSFZxPbpACdusVDPHXEjeEusYfk53N/KGHtdvrYo=,tag:ap38sCSTZVDQ0ZazXM3vlg==,type:str]
|
||||||
pgp: []
|
pgp: []
|
||||||
unencrypted_suffix: _unencrypted
|
unencrypted_suffix: _unencrypted
|
||||||
version: 3.9.1
|
version: 3.9.1
|
||||||
|
|
|
||||||
|
|
@ -10,20 +10,9 @@
|
||||||
# TODO: How to generate this key when bootstrap?
|
# TODO: How to generate this key when bootstrap?
|
||||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||||
secrets = {
|
secrets = {
|
||||||
github_public_token = {
|
"prometheus/metrics_username" = { };
|
||||||
owner = "root";
|
"prometheus/metrics_password" = { };
|
||||||
};
|
};
|
||||||
singbox_sg_server = {
|
|
||||||
owner = "root";
|
|
||||||
};
|
|
||||||
singbox_jp_server = {
|
|
||||||
owner = "root";
|
|
||||||
};
|
|
||||||
private_dns_address = {
|
|
||||||
owner = "root";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
secrets.grafana_cloud_api = lib.mkIf config.services.prometheus.enable { owner = "prometheus"; };
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
{ ... }:
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
./hardware-configurations.nix
|
./hardware-configurations.nix
|
||||||
|
./monitoring.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
config = {
|
||||||
networking.hostName = "thorite";
|
networking.hostName = "thorite";
|
||||||
networking.useNetworkd = true;
|
networking.useNetworkd = true;
|
||||||
systemd.network.enable = true;
|
systemd.network.enable = true;
|
||||||
|
|
@ -12,18 +13,28 @@
|
||||||
networkConfig = {
|
networkConfig = {
|
||||||
DHCP = "no";
|
DHCP = "no";
|
||||||
Gateway = "23.165.200.1";
|
Gateway = "23.165.200.1";
|
||||||
|
DNSSEC = true;
|
||||||
|
DNSOverTLS = true;
|
||||||
|
DNS = [
|
||||||
|
"8.8.8.8#dns.google"
|
||||||
|
"8.8.4.4#dns.google"
|
||||||
|
];
|
||||||
};
|
};
|
||||||
address = [ "23.165.200.99/24" ];
|
address = [ "23.165.200.99/24" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
nixpkgs.system = "x86_64-linux";
|
networking.firewall.allowedTCPPorts = [
|
||||||
|
80
|
||||||
system.stateVersion = "24.11";
|
443
|
||||||
|
];
|
||||||
|
|
||||||
commonSettings = {
|
commonSettings = {
|
||||||
auth.enable = true;
|
auth.enable = true;
|
||||||
autoupgrade.enable = true;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
nixpkgs.system = "x86_64-linux";
|
||||||
|
system.stateVersion = "24.11";
|
||||||
|
|
||||||
users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
|
users.users.root.hashedPassword = "$y$j9T$NToEZWJBONjSgRnMd9Ur9/$o6n7a9b8eUILQz4d37oiHCCVnDJ8hZTZt.c.37zFfU.";
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
103
machines/thorite/monitoring.nix
Normal file
103
machines/thorite/monitoring.nix
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
{ config, my-lib, ... }:
|
||||||
|
with my-lib;
|
||||||
|
{
|
||||||
|
config = {
|
||||||
|
sops = {
|
||||||
|
defaultSopsFile = ./secrets.yaml;
|
||||||
|
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||||
|
secrets = {
|
||||||
|
"grafana/oauth_secret" = {
|
||||||
|
owner = "grafana";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
custom.monitoring = {
|
||||||
|
grafana.enable = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
services.caddy.virtualHosts."https://grafana.xinyang.life".extraConfig =
|
||||||
|
with config.services.grafana.settings.server; ''
|
||||||
|
reverse_proxy http://${http_addr}:${toString http_port}
|
||||||
|
'';
|
||||||
|
|
||||||
|
custom.prometheus = {
|
||||||
|
enable = true;
|
||||||
|
exporters = {
|
||||||
|
enable = true;
|
||||||
|
blackbox.enable = true;
|
||||||
|
node.enable = true;
|
||||||
|
};
|
||||||
|
ruleModules = (mkCaddyRules [ { host = "thorite"; } ]) ++ (mkNodeRules [ { host = "thorite"; } ]);
|
||||||
|
};
|
||||||
|
|
||||||
|
services.prometheus.scrapeConfigs =
|
||||||
|
let
|
||||||
|
probeList = [
|
||||||
|
"la-00.video.namely.icu:8080"
|
||||||
|
"fre-00.video.namely.icu:8080"
|
||||||
|
"hk-00.video.namely.icu:8080"
|
||||||
|
"49.13.13.122:443"
|
||||||
|
"45.142.178.32:22"
|
||||||
|
"home.xinyang.life:8000"
|
||||||
|
];
|
||||||
|
passwordFile = config.sops.secrets."prometheus/metrics_password".path;
|
||||||
|
in
|
||||||
|
(mkScrapes [
|
||||||
|
{
|
||||||
|
name = "immich";
|
||||||
|
scheme = "http";
|
||||||
|
address = "weilite.coho-tet.ts.net";
|
||||||
|
port = 8082;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
inherit passwordFile;
|
||||||
|
name = "gotosocial";
|
||||||
|
address = "xinyang.life";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
inherit passwordFile;
|
||||||
|
name = "miniflux";
|
||||||
|
address = "rss.xinyang.life";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "ntfy";
|
||||||
|
address = "ntfy.xinyang.life";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "grafana-eu";
|
||||||
|
address = "grafana.xinyang.life";
|
||||||
|
}
|
||||||
|
])
|
||||||
|
++ (mkCaddyScrapes [
|
||||||
|
{ address = "thorite.coho-tet.ts.net"; }
|
||||||
|
])
|
||||||
|
++ (mkNodeScrapes [
|
||||||
|
{ address = "thorite.coho-tet.ts.net"; }
|
||||||
|
{ address = "massicot.coho-tet.ts.net"; }
|
||||||
|
{ address = "weilite.coho-tet.ts.net"; }
|
||||||
|
{ address = "hk-00.coho-tet.ts.net"; }
|
||||||
|
{ address = "la-00.coho-tet.ts.net"; }
|
||||||
|
{ address = "fra-00.coho-tet.ts.net"; }
|
||||||
|
])
|
||||||
|
++ (mkBlackboxScrapes [
|
||||||
|
{
|
||||||
|
hostAddress = "thorite.coho-tet.ts.net";
|
||||||
|
targetAddresses = probeList;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
hostAddress = "massicot.coho-tet.ts.net";
|
||||||
|
targetAddresses = probeList;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
hostAddress = "weilite.coho-tet.ts.net";
|
||||||
|
targetAddresses = [
|
||||||
|
"la-00.video.namely.icu:8080"
|
||||||
|
"fre-00.video.namely.icu:8080"
|
||||||
|
"hk-00.video.namely.icu:8080"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
|
||||||
|
};
|
||||||
|
}
|
||||||
31
machines/thorite/secrets.yaml
Normal file
31
machines/thorite/secrets.yaml
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
grafana:
|
||||||
|
oauth_secret: ENC[AES256_GCM,data:angZR3sl8vGcbAXyKFBvCSm+YhF5OooCcxRiSxR2zBoXMz5wv5/uMJFynwOTRVI6,iv:hVpOlM89lNbK6AsGf4Is/tLv3xPfg/XdtA8vuEK52L8=,tag:zCER+IdRnTcG2WHQ/AhxZA==,type:str]
|
||||||
|
sops:
|
||||||
|
kms: []
|
||||||
|
gcp_kms: []
|
||||||
|
azure_kv: []
|
||||||
|
hc_vault: []
|
||||||
|
age:
|
||||||
|
- recipient: age1uw059wcwfvd9xuj0hpqzqpeg7qemecspjrsatg37wc7rs2pumfdsgken0c
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoTXRtTlRES3V4MGhZaGdr
|
||||||
|
aXJ4UFNDT0Nrb0ZuWEkxUEFDU2orbzNBSVhVCkh2VitqMGwwOVdhMFJIeWU1eTgw
|
||||||
|
UVdxY0tLVDJNVnRnQmMyS0FPYS9LVmMKLS0tIEZaMTdIMU5SQUkxL2NFK2Jtbm9v
|
||||||
|
YVR3RHpDR3F2aFlCWGd5TjNOV2p4YzgK8OKpwcvTK/0j+kQCo0+8n6sQ5Pu9t9xZ
|
||||||
|
lPWeUGk1BudsyCqgIZWF5iXfu1pJnYq1XEAM0ttJl402xKeqIovM0Q==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age12ng08vjx5jde5ncqutwkd5vm4ygfwy33mzhzwe0lkxzglulgpqusc89r96
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIYk1NTlhsYW8xbFppNTBE
|
||||||
|
WlJmNzhnclBoVENXa1cvcHY5NGdRZVAzV0FJClpsTHpTeG9CK3J2ZFEreG1BTWpG
|
||||||
|
WjdaYzlLQnU0LzJLSDBZZ2pvOWdvSEkKLS0tIExRT0p1aCttZG5MMW14emJmRk5w
|
||||||
|
M2pqMUJoMGlBZnpBaVBUTFFRZUMzb2sKrlWy26Cv55/8XQEl9hee8P29uj582sIx
|
||||||
|
mUjaYE0U2qOP9bklXUQyyzQjfkBLWTLc1PTX9BjqOOsqXwkRQIYppA==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
lastmodified: "2024-11-28T17:02:03Z"
|
||||||
|
mac: ENC[AES256_GCM,data:14FOUXuKP+8+sad1UlhBW37fWzmutpyn6d4q2qKtBiOyT5ivHunFHJfHrtX83X2fLDmUfiD42bXf+rYfdtKzVUmQ6vutCUQk+Hal8NElhjcq5Ns5kT4VZRKG7/ya9+eNEEkajtq/7OFEM5KOQKTKjyOBqBq/AdYQ+ni9r45c1sM=,iv:WrdWSfrZrGalZO4WGk3JpgACY7W0odt3vP+pRkMXHfA=,tag:jeRBfR2QYjLBylOLHxU3hQ==,type:str]
|
||||||
|
pgp: []
|
||||||
|
unencrypted_suffix: _unencrypted
|
||||||
|
version: 3.9.1
|
||||||
|
|
@ -16,7 +16,6 @@
|
||||||
networking.hostName = "weilite";
|
networking.hostName = "weilite";
|
||||||
commonSettings = {
|
commonSettings = {
|
||||||
auth.enable = true;
|
auth.enable = true;
|
||||||
autoupgrade.enable = true;
|
|
||||||
nix = {
|
nix = {
|
||||||
enable = true;
|
enable = true;
|
||||||
enableMirrors = true;
|
enableMirrors = true;
|
||||||
|
|
@ -61,8 +60,14 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
custom.prometheus = {
|
custom.prometheus.exporters = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
blackbox = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
|
node = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.mounts = [
|
systemd.mounts = [
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
./disk-partitions
|
./disk-partitions
|
||||||
./restic.nix
|
./restic.nix
|
||||||
./vaultwarden.nix
|
./vaultwarden.nix
|
||||||
./prometheus
|
./monitor
|
||||||
./hedgedoc.nix
|
./hedgedoc.nix
|
||||||
./sing-box.nix
|
./sing-box.nix
|
||||||
./kanidm-client.nix
|
./kanidm-client.nix
|
||||||
|
|
|
||||||
171
modules/nixos/monitor/default.nix
Normal file
171
modules/nixos/monitor/default.nix
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
{
|
||||||
|
config,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
inherit (lib)
|
||||||
|
mkEnableOption
|
||||||
|
mkOption
|
||||||
|
mkIf
|
||||||
|
mkMerge
|
||||||
|
types
|
||||||
|
;
|
||||||
|
cfg = config.custom.prometheus;
|
||||||
|
|
||||||
|
mkRulesOption = mkOption {
|
||||||
|
type = types.listOf (
|
||||||
|
types.submodule {
|
||||||
|
options = {
|
||||||
|
name = mkOption { type = lib.types.str; };
|
||||||
|
rules = mkOption { type = lib.types.listOf lib.types.attrs; };
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
./exporters.nix
|
||||||
|
./grafana.nix
|
||||||
|
];
|
||||||
|
|
||||||
|
options = {
|
||||||
|
custom.monitoring = {
|
||||||
|
grafana = {
|
||||||
|
enable = mkEnableOption "grafana with oauth only";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
custom.prometheus = {
|
||||||
|
enable = mkEnableOption "Prometheus instance";
|
||||||
|
ruleModules = mkRulesOption;
|
||||||
|
exporters = {
|
||||||
|
enable = mkEnableOption "prometheus exporter on all supported and enable guarded services";
|
||||||
|
node = {
|
||||||
|
enable = mkEnableOption "node exporter";
|
||||||
|
listenAddress = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
default = "${config.networking.hostName}.coho-tet.ts.net";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
blackbox = {
|
||||||
|
enable = mkEnableOption "blackbox exporter";
|
||||||
|
listenAddress = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
default = "${config.networking.hostName}.coho-tet.ts.net";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = mkMerge [
|
||||||
|
{
|
||||||
|
sops.secrets = {
|
||||||
|
"prometheus/metrics_username" = {
|
||||||
|
sopsFile = ../../../machines/secrets.yaml;
|
||||||
|
group = "prometheus-auth";
|
||||||
|
mode = "0440";
|
||||||
|
};
|
||||||
|
|
||||||
|
"prometheus/metrics_password" = {
|
||||||
|
sopsFile = ../../../machines/secrets.yaml;
|
||||||
|
group = "prometheus-auth";
|
||||||
|
mode = "0440";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
users.groups.prometheus-auth.members = [
|
||||||
|
"prometheus"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
(mkIf cfg.enable {
|
||||||
|
|
||||||
|
services.tailscale = {
|
||||||
|
enable = true;
|
||||||
|
permitCertUid = config.services.caddy.user;
|
||||||
|
openFirewall = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
services.caddy = {
|
||||||
|
enable = true;
|
||||||
|
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
||||||
|
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
services.prometheus = mkIf cfg.enable {
|
||||||
|
enable = true;
|
||||||
|
port = 9091;
|
||||||
|
globalConfig.external_labels = {
|
||||||
|
hostname = config.networking.hostName;
|
||||||
|
};
|
||||||
|
|
||||||
|
scrapeConfigs = [
|
||||||
|
{
|
||||||
|
job_name = "prometheus";
|
||||||
|
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
alertmanager = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
logLevel = "debug";
|
||||||
|
configuration = {
|
||||||
|
route = {
|
||||||
|
receiver = "ntfy";
|
||||||
|
};
|
||||||
|
receivers = [
|
||||||
|
{
|
||||||
|
name = "ntfy";
|
||||||
|
webhook_configs = [
|
||||||
|
{
|
||||||
|
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
|
||||||
|
Alert {{.status}}
|
||||||
|
{{range .alerts}}-----{{range $k,$v := .labels}}
|
||||||
|
{{$k}}={{$v}}{{end}}
|
||||||
|
{{end}}
|
||||||
|
''}";
|
||||||
|
send_resolved = true;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
alertmanagers = [
|
||||||
|
{
|
||||||
|
scheme = "http";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
|
||||||
|
};
|
||||||
|
custom.prometheus.ruleModules = [
|
||||||
|
{
|
||||||
|
name = "prometheus_alerts";
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "JobDown";
|
||||||
|
expr = "up == 0";
|
||||||
|
for = "1m";
|
||||||
|
labels = {
|
||||||
|
severity = "critical";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "Job {{ $labels.job }} down for 1m.";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
})
|
||||||
|
];
|
||||||
|
}
|
||||||
102
modules/nixos/monitor/exporters.nix
Normal file
102
modules/nixos/monitor/exporters.nix
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
{
|
||||||
|
config,
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
inherit (lib) mkIf;
|
||||||
|
cfg = config.custom.prometheus.exporters;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
config = {
|
||||||
|
systemd.services.tailscaled.after =
|
||||||
|
(lib.optional cfg.node.enable "prometheus-node-exporters.service")
|
||||||
|
++ (lib.optional cfg.blackbox.enable "prometheus-blackbox-exporters.service")
|
||||||
|
++ (lib.optional config.services.caddy.enable "caddy.service");
|
||||||
|
|
||||||
|
services.prometheus.exporters.node = mkIf cfg.node.enable {
|
||||||
|
enable = true;
|
||||||
|
enabledCollectors = [
|
||||||
|
"loadavg"
|
||||||
|
"time"
|
||||||
|
"systemd"
|
||||||
|
];
|
||||||
|
listenAddress = cfg.node.listenAddress;
|
||||||
|
port = 9100;
|
||||||
|
};
|
||||||
|
|
||||||
|
services.prometheus.exporters.blackbox = mkIf cfg.blackbox.enable {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = cfg.blackbox.listenAddress;
|
||||||
|
configFile = pkgs.writeText "blackbox.config.yaml" (
|
||||||
|
lib.generators.toYAML { } {
|
||||||
|
modules = {
|
||||||
|
tcp4_connect = {
|
||||||
|
prober = "tcp";
|
||||||
|
tcp = {
|
||||||
|
ip_protocol_fallback = false;
|
||||||
|
preferred_ip_protocol = "ip4";
|
||||||
|
tls = false;
|
||||||
|
};
|
||||||
|
timeout = "15s";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
# gotosocial
|
||||||
|
sops.templates."gotosocial_metrics.env" = {
|
||||||
|
content = ''
|
||||||
|
GTS_METRICS_AUTH_ENABLED=true
|
||||||
|
GTS_METRICS_AUTH_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
|
||||||
|
GTS_METRICS_AUTH_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
|
||||||
|
'';
|
||||||
|
group = "prometheus-auth";
|
||||||
|
mode = "0440";
|
||||||
|
};
|
||||||
|
systemd.services.gotosocial.serviceConfig = {
|
||||||
|
EnvironmentFile = [ config.sops.templates."gotosocial_metrics.env".path ];
|
||||||
|
SupplementaryGroups = [ "prometheus-auth" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
services.gotosocial.settings = {
|
||||||
|
metrics-enabled = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
services.immich.environment = {
|
||||||
|
IMMICH_TELEMETRY_INCLUDE = "all";
|
||||||
|
};
|
||||||
|
|
||||||
|
services.restic.server.prometheus = true;
|
||||||
|
|
||||||
|
# miniflux
|
||||||
|
sops.templates."miniflux_metrics_env" = {
|
||||||
|
content = ''
|
||||||
|
METRICS_COLLECTOR=1
|
||||||
|
LOG_LEVEL=debug
|
||||||
|
METRICS_USERNAME=${config.sops.placeholder."prometheus/metrics_username"}
|
||||||
|
METRICS_PASSWORD=${config.sops.placeholder."prometheus/metrics_password"}
|
||||||
|
'';
|
||||||
|
group = "prometheus-auth";
|
||||||
|
mode = "0440";
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services.miniflux.serviceConfig = {
|
||||||
|
EnvironmentFile = [ config.sops.templates."miniflux_metrics_env".path ];
|
||||||
|
SupplementaryGroups = [ "prometheus-auth" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
services.ntfy-sh.settings.enable-metrics = true;
|
||||||
|
|
||||||
|
services.caddy.globalConfig = ''
|
||||||
|
servers {
|
||||||
|
metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
admin ${config.networking.hostName}.coho-tet.ts.net:2019 {
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
43
modules/nixos/monitor/grafana.nix
Normal file
43
modules/nixos/monitor/grafana.nix
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
{ config, lib, ... }:
|
||||||
|
let
|
||||||
|
cfg = config.custom.monitoring.grafana;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
sops.templates."grafana.env".content = ''
|
||||||
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${config.sops.placeholder."grafana/oauth_secret"}
|
||||||
|
'';
|
||||||
|
services.grafana = {
|
||||||
|
enable = true;
|
||||||
|
settings = {
|
||||||
|
server = {
|
||||||
|
http_addr = "127.0.0.1";
|
||||||
|
http_port = 3003;
|
||||||
|
root_url = "https://grafana.xinyang.life";
|
||||||
|
domain = "grafana.xinyang.life";
|
||||||
|
};
|
||||||
|
"auth.generic_oauth" = {
|
||||||
|
enabled = true;
|
||||||
|
name = "Kanidm";
|
||||||
|
client_id = "grafana";
|
||||||
|
scopes = "openid,profile,email,groups";
|
||||||
|
auth_url = "https://auth.xinyang.life/ui/oauth2";
|
||||||
|
token_url = "https://auth.xinyang.life/oauth2/token";
|
||||||
|
api_url = "https://auth.xinyang.life/oauth2/openid/grafana/userinfo";
|
||||||
|
use_pkce = true;
|
||||||
|
use_refresh_token = true;
|
||||||
|
allow_sign_up = true;
|
||||||
|
login_attribute_path = "preferred_username";
|
||||||
|
groups_attribute_path = "groups";
|
||||||
|
role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'";
|
||||||
|
allow_assign_grafana_admin = true;
|
||||||
|
auto_login = true;
|
||||||
|
};
|
||||||
|
"auth" = {
|
||||||
|
disable_login_form = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
systemd.services.grafana.serviceConfig.EnvironmentFile = config.sops.templates."grafana.env".path;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -3,17 +3,26 @@ let
|
||||||
cfg = config.custom.prometheus;
|
cfg = config.custom.prometheus;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.restic.enable) {
|
config = {
|
||||||
services.restic.server.prometheus = true;
|
services.restic.server.prometheus = true;
|
||||||
|
|
||||||
services.prometheus.scrapeConfigs = [
|
custom.prometheus.templates.scrape.mkResticScrapes =
|
||||||
|
{
|
||||||
|
address,
|
||||||
|
port ? null,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
portStr = if port then ":${toString port}" else "";
|
||||||
|
in
|
||||||
|
[
|
||||||
(lib.mkIf cfg.exporters.restic.enable {
|
(lib.mkIf cfg.exporters.restic.enable {
|
||||||
job_name = "restic";
|
job_name = "restic";
|
||||||
static_configs = [ { targets = [ config.services.restic.server.listenAddress ]; } ];
|
static_configs = [ { targets = [ "${address}${portStr}" ]; } ];
|
||||||
})
|
})
|
||||||
];
|
];
|
||||||
|
|
||||||
custom.prometheus.ruleModules = [
|
custom.prometheus.templates.rules.mkResticRules = [
|
||||||
{
|
{
|
||||||
name = "restic_alerts";
|
name = "restic_alerts";
|
||||||
rules = [
|
rules = [
|
||||||
|
|
@ -1,93 +0,0 @@
|
||||||
{
|
|
||||||
config,
|
|
||||||
lib,
|
|
||||||
pkgs,
|
|
||||||
...
|
|
||||||
}:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.blackbox.enable) {
|
|
||||||
services.prometheus.exporters.blackbox = {
|
|
||||||
enable = true;
|
|
||||||
listenAddress = "127.0.0.1";
|
|
||||||
configFile = pkgs.writeText "blackbox.config.yaml" (
|
|
||||||
lib.generators.toYAML { } {
|
|
||||||
modules = {
|
|
||||||
tcp4_connect = {
|
|
||||||
prober = "tcp";
|
|
||||||
tcp = {
|
|
||||||
ip_protocol_fallback = false;
|
|
||||||
preferred_ip_protocol = "ip4";
|
|
||||||
tls = false;
|
|
||||||
};
|
|
||||||
timeout = "15s";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "blackbox";
|
|
||||||
scrape_interval = "1m";
|
|
||||||
metrics_path = "/probe";
|
|
||||||
params = {
|
|
||||||
module = [ "tcp4_connect" ];
|
|
||||||
};
|
|
||||||
static_configs = [
|
|
||||||
{
|
|
||||||
targets = [
|
|
||||||
"tok-00.namely.icu:8080"
|
|
||||||
"la-00.video.namely.icu:8080"
|
|
||||||
"auth.xinyang.life:443"
|
|
||||||
"home.xinyang.life:8000"
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
relabel_configs = [
|
|
||||||
{
|
|
||||||
source_labels = [ "__address__" ];
|
|
||||||
target_label = "__param_target";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
source_labels = [ "__param_target" ];
|
|
||||||
target_label = "instance";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
target_label = "__address__";
|
|
||||||
replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "blackbox_exporter";
|
|
||||||
static_configs = [
|
|
||||||
{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
custom.prometheus.ruleModules = [
|
|
||||||
{
|
|
||||||
name = "probe_alerts";
|
|
||||||
rules = [
|
|
||||||
{
|
|
||||||
alert = "HighProbeLatency";
|
|
||||||
expr = "probe_duration_seconds > 0.5";
|
|
||||||
for = "2m";
|
|
||||||
labels = {
|
|
||||||
severity = "warning";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "High request latency on {{ $labels.instance }}";
|
|
||||||
description = "95th percentile of request latency is above 0.5 seconds for the last 2 minutes.";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
{ config, lib, ... }:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.caddy.enable) {
|
|
||||||
services.caddy.globalConfig = lib.mkIf cfg.exporters.caddy.enable ''
|
|
||||||
servers {
|
|
||||||
metrics
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "caddy";
|
|
||||||
static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
custom.prometheus.ruleModules = [
|
|
||||||
{
|
|
||||||
name = "caddy_alerts";
|
|
||||||
rules = [
|
|
||||||
{
|
|
||||||
alert = "UpstreamHealthy";
|
|
||||||
expr = "caddy_reverse_proxy_upstreams_healthy != 1";
|
|
||||||
for = "5m";
|
|
||||||
labels = {
|
|
||||||
severity = "critical";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "Upstream {{ $labels.unstream }} not healthy";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,230 +0,0 @@
|
||||||
{
|
|
||||||
config,
|
|
||||||
pkgs,
|
|
||||||
lib,
|
|
||||||
...
|
|
||||||
}:
|
|
||||||
|
|
||||||
with lib;
|
|
||||||
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
mkExporterOption =
|
|
||||||
enableOption:
|
|
||||||
(mkOption {
|
|
||||||
type = types.bool;
|
|
||||||
default = enableOption;
|
|
||||||
description = "Enable this exporter";
|
|
||||||
});
|
|
||||||
|
|
||||||
mkRulesOption = mkOption {
|
|
||||||
type = types.listOf (
|
|
||||||
types.submodule {
|
|
||||||
options = {
|
|
||||||
name = mkOption { type = lib.types.str; };
|
|
||||||
rules = mkOption { type = lib.types.listOf lib.types.attrs; };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
);
|
|
||||||
};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
imports = [
|
|
||||||
./blackbox.nix
|
|
||||||
./caddy.nix
|
|
||||||
./gotosocial.nix
|
|
||||||
./immich.nix
|
|
||||||
./miniflux.nix
|
|
||||||
./ntfy-sh.nix
|
|
||||||
./restic.nix
|
|
||||||
];
|
|
||||||
|
|
||||||
options = {
|
|
||||||
custom.prometheus = {
|
|
||||||
enable = mkEnableOption "Prometheus instance";
|
|
||||||
exporters = {
|
|
||||||
enable = mkOption {
|
|
||||||
type = types.bool;
|
|
||||||
default = false;
|
|
||||||
description = "Enable Prometheus exporter on every supported services";
|
|
||||||
};
|
|
||||||
|
|
||||||
restic.enable = mkExporterOption config.services.restic.server.enable;
|
|
||||||
blackbox.enable = mkExporterOption false;
|
|
||||||
caddy.enable = mkExporterOption config.services.caddy.enable;
|
|
||||||
gotosocial.enable = mkExporterOption config.services.gotosocial.enable;
|
|
||||||
immich.enable = mkExporterOption config.services.immich.enable;
|
|
||||||
miniflux.enable = mkExporterOption config.services.miniflux.enable;
|
|
||||||
ntfy-sh.enable = mkExporterOption config.services.ntfy-sh.enable;
|
|
||||||
};
|
|
||||||
grafana = {
|
|
||||||
enable = mkEnableOption "Grafana Cloud";
|
|
||||||
password_file = mkOption { type = types.path; };
|
|
||||||
};
|
|
||||||
ruleModules = mkRulesOption;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = mkIf cfg.enable {
|
|
||||||
services.tailscale = {
|
|
||||||
enable = true;
|
|
||||||
permitCertUid = config.services.caddy.user;
|
|
||||||
openFirewall = true;
|
|
||||||
};
|
|
||||||
|
|
||||||
services.caddy = {
|
|
||||||
enable = true;
|
|
||||||
virtualHosts."${config.networking.hostName}.coho-tet.ts.net".extraConfig = ''
|
|
||||||
reverse_proxy 127.0.0.1:${toString config.services.prometheus.port}
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
services.prometheus = mkIf cfg.enable {
|
|
||||||
enable = true;
|
|
||||||
port = 9091;
|
|
||||||
globalConfig.external_labels = {
|
|
||||||
hostname = config.networking.hostName;
|
|
||||||
};
|
|
||||||
remoteWrite = mkIf cfg.grafana.enable [
|
|
||||||
{
|
|
||||||
name = "grafana";
|
|
||||||
url = "https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push";
|
|
||||||
basic_auth = {
|
|
||||||
username = "1340065";
|
|
||||||
password_file = cfg.grafana.password_file;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
exporters = {
|
|
||||||
node = {
|
|
||||||
enable = true;
|
|
||||||
enabledCollectors = [
|
|
||||||
"loadavg"
|
|
||||||
"time"
|
|
||||||
"systemd"
|
|
||||||
];
|
|
||||||
listenAddress = "127.0.0.1";
|
|
||||||
port = 9100;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "prometheus";
|
|
||||||
static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.port}" ]; } ];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "node";
|
|
||||||
static_configs = [
|
|
||||||
{ targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
alertmanager = {
|
|
||||||
enable = true;
|
|
||||||
listenAddress = "127.0.0.1";
|
|
||||||
logLevel = "debug";
|
|
||||||
configuration = {
|
|
||||||
route = {
|
|
||||||
receiver = "ntfy";
|
|
||||||
};
|
|
||||||
receivers = [
|
|
||||||
{
|
|
||||||
name = "ntfy";
|
|
||||||
webhook_configs = [
|
|
||||||
{
|
|
||||||
url = "https://ntfy.xinyang.life/prometheus-alerts?tpl=yes&m=${lib.escapeURL ''
|
|
||||||
Alert {{.status}}
|
|
||||||
{{range .alerts}}-----{{range $k,$v := .labels}}
|
|
||||||
{{$k}}={{$v}}{{end}}
|
|
||||||
{{end}}
|
|
||||||
''}";
|
|
||||||
send_resolved = true;
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
alertmanagers = [
|
|
||||||
{
|
|
||||||
scheme = "http";
|
|
||||||
static_configs = [
|
|
||||||
{
|
|
||||||
targets = [
|
|
||||||
"${config.services.prometheus.alertmanager.listenAddress}:${toString config.services.prometheus.alertmanager.port}"
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
rules = [ (lib.generators.toYAML { } { groups = cfg.ruleModules; }) ];
|
|
||||||
};
|
|
||||||
custom.prometheus.ruleModules = [
|
|
||||||
{
|
|
||||||
name = "system_alerts";
|
|
||||||
rules = [
|
|
||||||
{
|
|
||||||
alert = "SystemdFailedUnits";
|
|
||||||
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
|
|
||||||
for = "5m";
|
|
||||||
labels = {
|
|
||||||
severity = "critical";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "Systemd has failed units on {{ $labels.instance }}";
|
|
||||||
description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
{
|
|
||||||
alert = "HighLoadAverage";
|
|
||||||
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
|
|
||||||
for = "1m";
|
|
||||||
labels = {
|
|
||||||
severity = "warning";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "High load average detected on {{ $labels.instance }}";
|
|
||||||
description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
{
|
|
||||||
alert = "HighTransmitTraffic";
|
|
||||||
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
|
|
||||||
for = "1m";
|
|
||||||
labels = {
|
|
||||||
severity = "warning";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
|
|
||||||
description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
{
|
|
||||||
alert = "NetworkTrafficExceedLimit";
|
|
||||||
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
|
|
||||||
for = "0m";
|
|
||||||
labels = {
|
|
||||||
severity = "critical";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "Outbound network traffic exceed 300GB for last 30 day";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
{
|
|
||||||
alert = "JobDown";
|
|
||||||
expr = "up == 0";
|
|
||||||
for = "1m";
|
|
||||||
labels = {
|
|
||||||
severity = "critical";
|
|
||||||
};
|
|
||||||
annotations = {
|
|
||||||
summary = "Job {{ $labels.job }} down for 1m.";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
{ config, lib, ... }:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.gotosocial.enable) {
|
|
||||||
services.gotosocial.settings = {
|
|
||||||
metrics-enabled = true;
|
|
||||||
};
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "gotosocial";
|
|
||||||
static_configs = [ { targets = [ "localhost:8080" ]; } ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
{ config, lib, ... }:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
immichEnv = config.services.immich.environment;
|
|
||||||
metricPort =
|
|
||||||
if builtins.hasAttr "IMMICH_API_METRICS_PORT" immichEnv then
|
|
||||||
immichEnv.IMMICH_API_METRICS_PORT
|
|
||||||
else
|
|
||||||
8081;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.immich.enable) {
|
|
||||||
services.immich.environment = {
|
|
||||||
IMMICH_METRICS = "true";
|
|
||||||
};
|
|
||||||
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "immich";
|
|
||||||
static_configs = [ { targets = [ "127.0.0.1:${toString metricPort}" ]; } ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
{ config, lib, ... }:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.miniflux.enable) {
|
|
||||||
systemd.services.miniflux.environment.METRICS_COLLECTOR = "1";
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "miniflux";
|
|
||||||
static_configs = [ { targets = [ config.systemd.services.miniflux.environment.LISTEN_ADDR ]; } ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
{ config, lib, ... }:
|
|
||||||
let
|
|
||||||
cfg = config.custom.prometheus;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (cfg.enable && cfg.exporters.ntfy-sh.enable) {
|
|
||||||
services.ntfy-sh.settings.enable-metrics = true;
|
|
||||||
services.prometheus.scrapeConfigs = [
|
|
||||||
{
|
|
||||||
job_name = "ntfy-sh";
|
|
||||||
static_configs = [ { targets = [ "ntfy.xinyang.life" ]; } ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
3
overlays/my-lib/default.nix
Normal file
3
overlays/my-lib/default.nix
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
}
|
||||||
|
// (import ./prometheus.nix)
|
||||||
209
overlays/my-lib/prometheus.nix
Normal file
209
overlays/my-lib/prometheus.nix
Normal file
|
|
@ -0,0 +1,209 @@
|
||||||
|
let
|
||||||
|
mkFunction = f: (targets: (map f targets));
|
||||||
|
mkPort = port: if isNull port then "" else ":${toString port}";
|
||||||
|
in
|
||||||
|
{
|
||||||
|
mkScrapes = mkFunction (
|
||||||
|
{
|
||||||
|
name,
|
||||||
|
address,
|
||||||
|
passwordFile ? null,
|
||||||
|
port ? 443,
|
||||||
|
scheme ? "https",
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
job_name = "${name}(${address})";
|
||||||
|
scheme = scheme;
|
||||||
|
static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
|
||||||
|
}
|
||||||
|
// (
|
||||||
|
if isNull passwordFile then
|
||||||
|
{ }
|
||||||
|
else
|
||||||
|
{
|
||||||
|
basic_auth.username = "prom";
|
||||||
|
basic_auth.password_file = passwordFile;
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
mkCaddyScrapes = mkFunction (
|
||||||
|
{
|
||||||
|
address,
|
||||||
|
port ? 2019,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
job_name = "caddy_${address}";
|
||||||
|
static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
mkCaddyRules = mkFunction (
|
||||||
|
{
|
||||||
|
host ? "",
|
||||||
|
name ? "caddy_alerts_${host}",
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
inherit name;
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "UpstreamHealthy";
|
||||||
|
expr = "caddy_reverse_proxy_upstreams_healthy != 1";
|
||||||
|
for = "5m";
|
||||||
|
labels = {
|
||||||
|
severity = "critical";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "Upstream {{ $labels.unstream }} not healthy";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
mkNodeScrapes = mkFunction (
|
||||||
|
{
|
||||||
|
address,
|
||||||
|
port ? 9100,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
job_name = "node_${address}";
|
||||||
|
static_configs = [ { targets = [ "${address}${mkPort port}" ]; } ];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
mkNodeRules = mkFunction (
|
||||||
|
{
|
||||||
|
host ? "",
|
||||||
|
name ? "system_alerts_${host}",
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
inherit name;
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "SystemdFailedUnits";
|
||||||
|
expr = "node_systemd_unit_state{state=\"failed\"} > 0";
|
||||||
|
for = "5m";
|
||||||
|
labels = {
|
||||||
|
severity = "critical";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "Systemd has failed units on {{ $labels.instance }}";
|
||||||
|
description = "There are {{ $value }} failed units on {{ $labels.instance }}. Immediate attention required!";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "HighLoadAverage";
|
||||||
|
expr = "node_load1 > 0.8 * count without (cpu) (node_cpu_seconds_total{mode=\"idle\"})";
|
||||||
|
for = "1m";
|
||||||
|
labels = {
|
||||||
|
severity = "warning";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "High load average detected on {{ $labels.instance }}";
|
||||||
|
description = "The 1-minute load average ({{ $value }}) exceeds 80% the number of CPUs.";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "HighTransmitTraffic";
|
||||||
|
expr = "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) > 100000000";
|
||||||
|
for = "1m";
|
||||||
|
labels = {
|
||||||
|
severity = "warning";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "High network transmit traffic on {{ $labels.instance }} ({{ $labels.device }})";
|
||||||
|
description = "The network interface {{ $labels.device }} on {{ $labels.instance }} is transmitting data at a rate exceeding 100 MB/s for the last 1 minute.";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "NetworkTrafficExceedLimit";
|
||||||
|
expr = ''increase(node_network_transmit_bytes_total{device!="lo",device!~"tailscale.*",device!~"wg.*",device!~"br.*"}[30d]) > 322122547200'';
|
||||||
|
for = "0m";
|
||||||
|
labels = {
|
||||||
|
severity = "critical";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "Outbound network traffic exceed 300GB for last 30 day";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
mkBlackboxScrapes = mkFunction (
|
||||||
|
{
|
||||||
|
hostAddress,
|
||||||
|
hostPort ? 9115,
|
||||||
|
targetAddresses,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
job_name = "blackbox(${hostAddress})";
|
||||||
|
scrape_interval = "1m";
|
||||||
|
metrics_path = "/probe";
|
||||||
|
params = {
|
||||||
|
module = [ "tcp4_connect" ];
|
||||||
|
};
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = targetAddresses;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
relabel_configs = [
|
||||||
|
{
|
||||||
|
source_labels = [ "__address__" ];
|
||||||
|
target_label = "__param_target";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
source_labels = [ "__param_target" ];
|
||||||
|
target_label = "instance";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
target_label = "__address__";
|
||||||
|
replacement = "${hostAddress}${mkPort hostPort}";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
mkBlackboxRules = mkFunction (
|
||||||
|
{
|
||||||
|
host ? "",
|
||||||
|
name ? "probe_alerts_${host}",
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
inherit name;
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
alert = "HighProbeLatency";
|
||||||
|
expr = "probe_duration_seconds > 0.5";
|
||||||
|
for = "3m";
|
||||||
|
labels = {
|
||||||
|
severity = "warning";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "High request latency on {{ $labels.instance }}";
|
||||||
|
description = "Request latency is above 0.5 seconds for the last 3 minutes.";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
alert = "VeryHighProbeLatency";
|
||||||
|
expr = "probe_duration_seconds > 1";
|
||||||
|
for = "3m";
|
||||||
|
labels = {
|
||||||
|
severity = "critical";
|
||||||
|
};
|
||||||
|
annotations = {
|
||||||
|
summary = "High request latency on {{ $labels.instance }}";
|
||||||
|
description = "Request latency is above 0.5 seconds for the last 3 minutes.";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue