[CI][Benchmarks] Fix umf benchmarks failures (#19388)

1. The `umf` benchmarks uses `GBench` to create a csv from the
benhcmark's output. Mixing up `stdout` with `stderr` led to a resulting
csv header resolvment to fail.

2. Fail the scripts also when any suite setup fails with
`--exit-on-failure` option enabled

3. Abort scripts when no benchmarks are defined.
Don't allow for creating html and markdown files in case it's not a dry
run and no benchmarks were properly set up.

Author: PatKamin

devops/scripts/benchmarks/main.py

@@ -221,6 +221,8 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
             try:
                 s.setup()
             except Exception as e:
+                if options.exit_on_failure:
+                    raise e
                 failures[s.name()] = f"Suite setup failure: {e}"
                 log.error(
                     f"{type(s).__name__} setup failed. Benchmarks won't be added."
@@ -247,7 +249,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
     if benchmarks:
         log.info(f"Running {len(benchmarks)} benchmarks...")
     elif not options.dry_run:
-        log.warning("No benchmarks to run.")
+        raise RuntimeError("No benchmarks to run.")
     for benchmark in benchmarks:
         try:
             merged_env_vars = {**additional_env_vars}

devops/scripts/benchmarks/utils/utils.py

@@ -65,17 +65,20 @@ def run(
             cwd=cwd,
             check=True,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
+            stderr=subprocess.PIPE,
             env=env,
             timeout=timeout,
         )  # nosec B603
 
         if result.stdout:
             log.debug(result.stdout.decode())
+        if result.stderr:
+            log.debug(result.stderr.decode())
 
         return result
     except subprocess.CalledProcessError as e:
         log.error(e.stdout.decode())
+        log.error(e.stderr.decode())
         raise