Add "invert_result" to test() option

docs/markdown/snippets/invert_result.md

@@ -0,0 +1,25 @@
+## `invert_result` option added to test()
+
+This is very similar to "should_fail", but does not change the test results to
+"Expected Fail" / "Unexpected Pass".
+
+"Expected Fail" and "Unexpected Pass" are useful when they communicate to
+developers that additional work needs to be done, e.g. as part of Test
+Driven Development or to demonstrate problems with some edge cases that need
+to be addressed in the future.
+
+In other cases, a non-zero status code is actually the whole test, e.g. a
+program under test that is expected to exit with a non-zero status code when
+given invalid arguments or a bad configuration file. This is the expected
+behavior and hence should be reported as "Pass", not "Expected Fail".
+
+"invert_result" and "should_fail" can be combined to produce "Unexpected Pass"
+for non-zero status codes and "Expected Fail" for zero status codes.
+
+```meson
+test(
+    'fail if config file is missing',
+    executable(...),
+    args: [ '--config', '/path/does/not/exist' ],
+    invert_result: true)
+```

docs/yaml/functions/benchmark.yaml

@@ -43,7 +43,19 @@ kwargs:
     default: false
     description: |
       when true the test is considered passed if the
-      executable returns a non-zero return value (i.e. reports an error)
+      executable returns a non-zero return value (i.e. reports an error).
+      This replaces the test result "Ok" with "Unexpected Pass" and "Fail"
+      with "Expected Fail".
+
+  invert_result:
+    type: bool
+    default: false
+    since: 1.8
+    description: |
+      when true the test is considered passed if the executable returns a
+      non-zero return value (i.e. reports an error). When used together with
+      `should_fail` non-zero return values are reported as "Unexpected Pass"
+      and zero return values as "Expected Fail".
 
   suite:
     type: str | list[str]

mesonbuild/backend/backends.py

@@ -200,6 +200,7 @@ class TestSerialisation:
     cmd_args: T.List[str]
     env: mesonlib.EnvironmentVariables
     should_fail: bool
+    invert_result: bool
     timeout: T.Optional[int]
     workdir: T.Optional[str]
     extra_paths: T.List[str]
@@ -1301,7 +1302,7 @@ def create_test_serialisation(self, tests: T.List['Test']) -> T.List[TestSeriali
             ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross,
                                    exe_wrapper, self.environment.need_exe_wrapper(),
                                    t.is_parallel, cmd_args, t_env,
-                                   t.should_fail, t.timeout, t.workdir,
+                                   t.should_fail, t.invert_result, t.timeout, t.workdir,
                                    extra_paths, t.protocol, t.priority,
                                    isinstance(exe, (build.Target, build.CustomTargetIndex)),
                                    isinstance(exe, build.Executable),

mesonbuild/interpreter/interpreter.py

@@ -2303,6 +2303,7 @@ def make_test(self, node: mparser.BaseNode,
                     kwargs['args'],
                     env,
                     kwargs['should_fail'],
+                    kwargs['invert_result'],
                     kwargs['timeout'],
                     kwargs['workdir'],
                     kwargs['protocol'],

mesonbuild/interpreter/interpreterobjects.py

@@ -785,7 +785,7 @@ def __init__(self, name: str, project: str, suite: T.List[str],
                  is_parallel: bool,
                  cmd_args: T.List[T.Union[str, mesonlib.File, build.Target, ExternalProgram]],
                  env: mesonlib.EnvironmentVariables,
-                 should_fail: bool, timeout: int, workdir: T.Optional[str], protocol: str,
+                 should_fail: bool, invert_result: bool, timeout: int, workdir: T.Optional[str], protocol: str,
                  priority: int, verbose: bool):
         super().__init__()
         self.name = name
@@ -797,6 +797,7 @@ def __init__(self, name: str, project: str, suite: T.List[str],
         self.cmd_args = cmd_args
         self.env = env
         self.should_fail = should_fail
+        self.invert_result = invert_result
         self.timeout = timeout
         self.workdir = workdir
         self.protocol = TestProtocol.from_str(protocol)

mesonbuild/interpreter/kwargs.py

@@ -40,6 +40,7 @@ class BaseTest(TypedDict):
 
     args: T.List[T.Union[str, File, build.Target, ExternalProgram]]
     should_fail: bool
+    invert_result: bool
     timeout: int
     workdir: T.Optional[str]
     depends: T.List[T.Union[build.CustomTarget, build.BuildTarget]]

mesonbuild/interpreter/type_checking.py

@@ -488,6 +488,7 @@ def link_whole_validator(values: T.List[T.Union[StaticLibrary, CustomTarget, Cus
     KwargInfo('args', ContainerTypeInfo(list, (str, File, BuildTarget, CustomTarget, CustomTargetIndex, ExternalProgram)),
               listify=True, default=[]),
     KwargInfo('should_fail', bool, default=False),
+    KwargInfo('invert_result', bool, default=False, since='1.8'),
     KwargInfo('timeout', int, default=30),
     KwargInfo('workdir', (str, NoneType), default=None,
               validator=lambda x: 'must be an absolute path' if not os.path.isabs(x) else None),

mesonbuild/mtest.py

@@ -941,6 +941,7 @@ def __init__(self, test: TestSerialisation, test_env: T.Dict[str, str],
         self.cmd: T.Optional[T.List[str]] = None
         self.env = test_env
         self.should_fail = test.should_fail
+        self.invert_result = test.invert_result
         self.project = test.project_name
         self.junit: T.Optional[et.ElementTree] = None
         self.is_parallel = is_parallel
@@ -988,8 +989,11 @@ def _complete(self) -> None:
         if self.res == TestResult.RUNNING:
             self.res = TestResult.OK
         assert isinstance(self.res, TestResult)
-        if self.should_fail and self.res in (TestResult.OK, TestResult.FAIL):
-            self.res = TestResult.UNEXPECTEDPASS if self.res is TestResult.OK else TestResult.EXPECTEDFAIL
+        if self.res in (TestResult.OK, TestResult.FAIL):
+            if self.invert_result:
+                self.res = TestResult.FAIL if self.res is TestResult.OK else TestResult.OK
+            if self.should_fail:
+                self.res = TestResult.UNEXPECTEDPASS if self.res is TestResult.OK else TestResult.EXPECTEDFAIL
         if self.stdo and not self.stdo.endswith('\n'):
             self.stdo += '\n'
         if self.stde and not self.stde.endswith('\n'):

test cases/common/281 inverted result/meson.build

@@ -0,0 +1,4 @@
+project('inverted result', 'c')
+
+test('must_fail_and_does', executable('ret1', 'ret1.c'),
+     invert_result: true)

test cases/common/281 inverted result/ret1.c

@@ -0,0 +1 @@
+int main() { return 1; }

test cases/common/282 inverted result with should fail/meson.build

@@ -0,0 +1,5 @@
+project('inverted result', 'c')
+
+test('must_not_fail_and_does_not', executable('ret0', 'ret0.c'),
+     should_fail: true,
+     invert_result: true)

test cases/common/282 inverted result with should fail/ret0.c

@@ -0,0 +1 @@
+int main() { return 0; }

test cases/failing test/7 inverted result/meson.build

@@ -0,0 +1,4 @@
+project('inverted result', 'c')
+
+test('must_fail_but_does_not', executable('ret0', 'ret0.c'),
+     invert_result: true)

test cases/failing test/7 inverted result/ret0.c

@@ -0,0 +1 @@
+int main() { return 0; }

test cases/failing test/8 inverted result with should fail/meson.build

@@ -0,0 +1,5 @@
+project('inverted result', 'c')
+
+test('must_not_fail_but_does', executable('ret1', 'ret1.c'),
+     should_fail: true,
+     invert_result: true)

test cases/failing test/8 inverted result with should fail/ret1.c

@@ -0,0 +1 @@
+int main() { return 1; }