Warn if frameworks exceed the maximum number of test mutations

> The number of test mutations will be limited to 10.
#8420

This change shows a warning message for frameworks if the maximum is reached.

Author: p8

toolset/utils/metadata.py

@@ -191,6 +191,18 @@ def parse_config(self, config, directory):
                     % config['framework'],
                     color=Fore.YELLOW)
 
+            # Check that each framework does not have more than the maximum number of tests
+            maximum_tests = 10
+            non_broken_tests_filter = lambda test: (not hasattr(test, "tags")) or ("broken" not in test.tags)
+            non_broken_tests_to_run = list(filter(non_broken_tests_filter, tests_to_run))
+            if len(non_broken_tests_to_run) > maximum_tests:
+                message = [
+                    "Framework %s defines %s tests in benchmark_config.json (max is %s)."
+                        % (config['framework'], len(non_broken_tests_to_run), maximum_tests),
+                    "Contact maintainers and remove deprecated or discarded ones to make room."
+                ]
+                log("\n".join(message), color=Fore.YELLOW)
+
             # Check that each test configuration is acceptable
             # Throw exceptions if a field is missing, or how to improve the field
             for test_name, test_keys in test.items():