Py tooling: ignore skipped/error benchmarks (#2290)

* Ignore skipped benchmark when computing geomean

Skipped benchmarks have their real and CPU times set to zero. If they are part of the data for which the geomean is calculated, these zeros cause the computed geomean to be zero. Hence, a single skipped benchmark will effectively disable the computation of a geomean.

This seems unfortunate as in many cases there are lots of other benchmarks that still ran normally and for which one might still be interested in the overall geomean change.

* Use 'skipped' attribute instead of 'skip_message'

* Remove skipped benchmarks from partitions

* Add skipped benchmark to test data

* Also skip/ignore benchmarks with errors

* Pacify python pre-commit check

---------

Co-authored-by: Roman Lebedev <lebedev.ri@gmail.com>

tools/gbench/Inputs/test1_run1.json

diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json
index 9daed0b..b4871eb 100644
--- a/tools/gbench/Inputs/test1_run1.json
+++ b/tools/gbench/Inputs/test1_run1.json
@@ -122,6 +122,25 @@
       "real_time": 1,
       "cpu_time": 1,
       "time_unit": "s"
+    },
+    {
+      "name": "BM_skipped",
+      "label": "a label",
+      "iterations": 1,
+      "real_time": 0,
+      "cpu_time": 0,
+      "time_unit": "s",
+	  "skipped": true,
+      "skip_message": "Some message"
+    },
+    {
+      "name": "BM_error",
+      "iterations": 1,
+      "real_time": 0,
+      "cpu_time": 0,
+      "time_unit": "s",
+	  "error_occurred": true,
+      "error_message": "Something happened"
     }
   ]
 }

tools/gbench/Inputs/test1_run2.json

diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json
index dc52970..75e4708 100644
--- a/tools/gbench/Inputs/test1_run2.json
+++ b/tools/gbench/Inputs/test1_run2.json
@@ -122,6 +122,25 @@
       "real_time": 1,
       "cpu_time": 1,
       "time_unit": "s"
+    },
+    {
+      "name": "BM_skipped",
+      "label": "a label",
+      "iterations": 1,
+      "real_time": 0,
+      "cpu_time": 0,
+      "time_unit": "s",
+	  "skipped": true,
+      "skip_message": "Some message"
+    },
+    {
+      "name": "BM_error",
+      "iterations": 1,
+      "real_time": 0,
+      "cpu_time": 0,
+      "time_unit": "s",
+	  "error_occurred": true,
+      "error_message": "Something happened"
     }
   ]
 }

tools/gbench/report.py

diff --git a/tools/gbench/report.py b/tools/gbench/report.py
index e143e45..16b674a 100644
--- a/tools/gbench/report.py
+++ b/tools/gbench/report.py
@@ -161,13 +161,23 @@
         lhs = [
             x
             for x in json1["benchmarks"]
-            if x["name"] == name and x["time_unit"] == time_unit
+            if x["name"] == name
+            and x["time_unit"] == time_unit
+            and not x.get("skipped", False)
+            and not x.get("error_occurred", False)
         ]
         rhs = [
             x
             for x in json2["benchmarks"]
-            if x["name"] == name and x["time_unit"] == time_unit
+            if x["name"] == name
+            and x["time_unit"] == time_unit
+            and not x.get("skipped", False)
+            and not x.get("error_occurred", False)
         ]
+
+        if len(lhs) == 0 or len(rhs) == 0:
+            continue
+
         partitions.append([lhs, rhs])
     return partitions
 
@@ -191,6 +201,10 @@
     for benchmark in json["benchmarks"]:
         if "run_type" in benchmark and benchmark["run_type"] == "aggregate":
             continue
+        if benchmark.get("skipped", False) or benchmark.get(
+            "error_occurred", False
+        ):
+            continue
         times.append(
             [
                 get_timedelta_field_as_seconds(benchmark, "real_time"),