diff --git a/nixos/tests/caddy.nix b/nixos/tests/caddy.nix
index 500c9d7d94d1b4..21022b7992c6fd 100644
--- a/nixos/tests/caddy.nix
+++ b/nixos/tests/caddy.nix
@@ -89,6 +89,26 @@ import ./make-test-python.nix (
               '';
             };
           };
+          specialisation.invalid-config-warning.configuration = {
+            services.caddy = {
+              enforceConfigValidation = false;
+              extraConfig = ''
+                not-a-real-directive {
+                  this-should-warn
+                }
+              '';
+            };
+          };
+          specialisation.invalid-config-fail.configuration = {
+            services.caddy = {
+              enforceConfigValidation = true;
+              extraConfig = ''
+                not-a-real-directive {
+                  this-should-fail
+                }
+              '';
+            };
+          };
         };
     };
 
@@ -101,6 +121,8 @@ import ./make-test-python.nix (
         multipleHostnames = "${nodes.webserver.system.build.toplevel}/specialisation/multiple-hostnames";
         rfc42Config = "${nodes.webserver.system.build.toplevel}/specialisation/rfc42";
         withPluginsConfig = "${nodes.webserver.system.build.toplevel}/specialisation/with-plugins";
+        invalidConfigWarning = "${nodes.webserver.system.build.toplevel}/specialisation/invalid-config-warning";
+        invalidConfigFail = "${nodes.webserver.system.build.toplevel}/specialisation/invalid-config-fail";
       in
       ''
         url = "http://localhost/example.html"
@@ -150,6 +172,17 @@ import ./make-test-python.nix (
             )
             webserver.wait_for_open_port(80)
             webserver.succeed("curl http://localhost | grep caddy")
+
+        with subtest("invalid config shows warning when validation is not enforced"):
+            webserver.succeed(
+                "${invalidConfigWarning}/bin/switch-to-configuration test >&2"
+            )
+            webserver.wait_for_unit("caddy")
+
+        with subtest("invalid config fails build when validation is enforced"):
+            webserver.fail(
+                "${invalidConfigFail}/bin/switch-to-configuration test >&2"
+            )
       '';
   }
 )