logging and ddos protection

hosts/nixos/HandlerOne/default.nix

@@ -52,6 +52,8 @@
       "modules/services/ntfy-sh"
       "modules/services/ollama/nginx.nix" # Just host the nginx path back to Parzival
       "modules/services/netbox"
+      "modules/services/system-logging"
+      "modules/services/system-logging/nginx.nix"
       "modules/services/flamesites"
     ])
   ];
@@ -127,6 +129,7 @@
       "chat.wanderingcrow.net" = {};
       "netbox.wanderingcrow.net" = {};
       "notify.wanderingcrow.net" = {};
+      "logs.wanderingcrow.net" = {};
       # Sites I host for someone else
       "swgalaxyproject.com" = {};
       "nnsbluegrass.com" = {};

modules/services/fail2ban/default.nix

@@ -9,33 +9,36 @@
     "fail2ban/action.d/ntfy.local".text = pkgs.lib.mkDefault (pkgs.lib.mkAfter ''
       [Definition]
       norestored = true # Needed to avoid receiving a new notification after every restart
-      actionban = curl -H "Title: <ip> has been banned" -d "<name> jail has banned <ip> from accessing ${config.hostSpec.hostName} after <failures> attempts of hacking the system." https://notify.wanderingcrow.net/Fail2banNotifications
+      actionban = curl -H "Title: <ip> has been banned" -d "<name> jail has banned <ip> from accessing ${config.hostSpec.hostName} after <failures> attempts of attacking the system." https://notify.wanderingcrow.net/Fail2banNotifications
     '');
     # Defines a filter that detects URL probing by reading the Nginx access log
-    "fail2ban/filter.d/nginx-url-probe.local".text = pkgs.lib.mkDefault (pkgs.lib.mkAfter ''
+    "fail2ban/filter.d/nginx-ddos-protect.local".text = pkgs.lib.mkDefault (pkgs.lib.mkAfter ''
       [Definition]
-      failregex = ^<HOST>.*(GET /(admin|boaform|phpmyadmin|\.env|\.git)|\.(dll|so|cfm|asp)|(\?|&)(=PHPB8B5F2A0-3C92-11d3-A3A9-4C7B08C10000|=PHPE9568F36-D428-11d2-A769-00AA001ACF42|=PHPE9568F35-D428-11d2-A769-00AA001ACF42|=PHPE9568F34-D428-11d2-A769-00AA001ACF42)|\\x[0-9a-zA-Z]{2})
+      failregex = ^<HOST> - - \[.*\] "GET .* HTTP/.*" \d{3} \d+ "-" "ApacheBench/.+"
+                  limiting requests, excess:.* by zone.*client: <HOST>
+                  ^.*\[error\]\s+\d+#\d+:\s+\*\d+\s+connect\(\) to unix:.*failed.*while connecting to upstream,\s+client:\s+<HOST>,\s+server:.*
+                  ^.*\[error\]\s+\d+#\d+:\s+\*\d+\s+upstream prematurely closed connection while reading response header from upstream,\s+client:\s+<HOST>,.*$
+      ignoreregex =
     '');
   };
   services.fail2ban = {
-    enable = false;
+    enable = true;
     extraPackages = [
       pkgs.curl
     ];
     ignoreIP = [
       inputs.nix-secrets.network.primary.publicIP
-      "64.189.142.0/23"
     ];
     jails = {
-      nginx-url-probe.settings = {
+      nginx-ddos-protect.settings = {
         enabled = true;
-        filter = "nginx-url-probe";
-        logpath = "/var/log/nginx/access.log";
+        filter = "nginx-ddos-protect";
+        logpath = "/var/log/nginx/*.log";
         action = ''          %(action_)s[blocktype=DROP]
                            ntfy'';
         backend = "auto"; # Do not forget to specify this if your jail uses a log file
-        maxretry = 5;
-        findtime = 600;
+        maxretry = 1;
+        findtime = 3600;
       };
     };
   };

modules/services/system-logging/default.nix

@@ -0,0 +1,64 @@
+{
+  pkgs,
+  config,
+  ...
+}: let
+  lokiConfig = pkgs.writeText "lokiconfig" ''
+    # This is a complete configuration to deploy Loki backed by the filesystem.
+    # The index will be shipped to the storage via tsdb-shipper.
+
+    auth_enabled: false
+
+    server:
+      http_listen_port: 3100
+
+    common:
+      ring:
+        instance_addr: 127.0.0.1
+        kvstore:
+          store: inmemory
+      replication_factor: 1
+      path_prefix: /tmp/loki
+
+    schema_config:
+      configs:
+      - from: 2020-05-15
+        store: tsdb
+        object_store: filesystem
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+
+    storage_config:
+      filesystem:
+        directory: /tmp/loki/chunks
+  '';
+
+  alloyConfig = pkgs.writeText "alloyconfig" '''';
+in {
+  services = {
+    grafana = {
+      enable = true;
+      settings = {
+        analytics.reporting_enabled = false;
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 2432;
+          enable_gzip = true;
+          domain = "logs.wanderingcrow.net";
+          #root_url = "https://logs.wanderingcrow.net/${config.hostSpec.hostName}";
+          #serve_from_subpath = false;
+        };
+      };
+    };
+    loki = {
+      enable = true;
+      configFile = lokiConfig;
+    };
+    alloy = {
+      enable = true;
+      configPath = alloyConfig;
+    };
+  };
+}

modules/services/system-logging/nginx.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  inputs,
+  ...
+}: {
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    virtualHosts = {
+      "logs.wanderingcrow.net" = {
+        forceSSL = true;
+        useACMEHost = "logs.wanderingcrow.net";
+        locations."/" = {
+          extraConfig = ''
+            allow 192.168.0.0/16;
+            allow ${inputs.nix-secrets.network.primary.publicIP};
+            deny all;
+          '';
+          proxyPass = "http://${builtins.toString config.services.grafana.settings.server.http_addr}:${builtins.toString config.services.grafana.settings.server.http_port}";
+          proxyWebsockets = true;
+          recommendedProxySettings = true;
+        };
+      };
+    };
+  };
+}