Support testing that we get the right set of asserts
diff --git a/resources/test/conftest.py b/resources/test/conftest.py index 1baf662..168daa1 100644 --- a/resources/test/conftest.py +++ b/resources/test/conftest.py
@@ -114,7 +114,11 @@ continue if element.tag == 'script': if element.attrib.get('id') == 'expected': - self.expected = json.loads(text_type(element.text)) + try: + self.expected = json.loads(text_type(element.text)) + except ValueError: + print("Failed parsing JSON in %s" % filename) + raise src = element.attrib.get('src', '') @@ -192,6 +196,8 @@ test_url = self.url + variant actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % test_url) + print(json.dumps(actual, indent=2)) + summarized = self._summarize(copy.deepcopy(actual)) print(json.dumps(summarized, indent=2)) @@ -203,6 +209,17 @@ self.expected[u'summarized_tests'].sort(key=lambda test_obj: test_obj.get('name')) + # Make asserts opt-in for now + if "summarized_asserts" not in self.expected: + del summarized["summarized_asserts"] + else: + # We can't be sure of the order of asserts even within the same test + # although we could also check for the failing assert being the final + # one + for obj in [summarized, self.expected]: + obj["summarized_asserts"].sort( + key=lambda x: (x["test"], x["status"], x["assert_name"], tuple(x["args"]))) + assert summarized == self.expected def _summarize(self, actual): @@ -212,6 +229,11 @@ summarized[u'summarized_tests'] = [ self._summarize_test(test) for test in actual['tests']] summarized[u'summarized_tests'].sort(key=lambda test_obj: test_obj.get('name')) + summarized[u'summarized_asserts'] = [ + {"assert_name": assert_item["assert_name"], + "test": assert_item["test"]["name"] if assert_item["test"] else None, + "args": assert_item["args"], + "status": assert_item["status"]} for assert_item in actual["asserts"]] summarized[u'type'] = actual['type'] return summarized
diff --git a/resources/test/tests/functional/api-tests-1.html b/resources/test/tests/functional/api-tests-1.html index 237b4ea..41a765f 100644 --- a/resources/test/tests/functional/api-tests-1.html +++ b/resources/test/tests/functional/api-tests-1.html
@@ -562,6 +562,431 @@ "properties": {} } ], + "summarized_asserts": [ + { + "assert_name": "assert_true", + "test": "Setup function ran", + "args": [ + "true" + ], + "status": 0 + }, + { + "assert_name": "assert_array_equals", + "test": "on_event does not use event capture", + "args": [ + "[\"inner\", \"outer\"]", + "[\"inner\", \"outer\"]" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "document.body should be the first body element in the document", + "args": [ + "Element node <body onload=\"load_test_attr.done()\"> <h1>Sample HTML5 AP...", + "Element node <body onload=\"load_test_attr.done()\"> <h1>Sample HTML5 AP..." + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "assert_equals tests", + "args": [ + "1", + "1" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "assert_equals tests", + "args": [ + "NaN", + "NaN", + "\"NaN case\"" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "assert_equals tests", + "args": [ + "0", + "0", + "\"Zero case\"" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "assert_equals tests expected to fail", + "args": [ + "-0", + "0", + "\"Zero case\"" + ], + "status": 1 + }, + { + "assert_name": "assert_not_equals", + "test": "assert_not_equals tests", + "args": [ + "object \"[object Object]\"", + "object \"[object Object]\"", + "\"object case\"" + ], + "status": 0 + }, + { + "assert_name": "assert_not_equals", + "test": "assert_not_equals tests", + "args": [ + "-0", + "0", + "\"Zero case\"" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "assert_true expected to pass", + "args": [ + "true" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "assert_true expected to fail", + "args": [ + "false", + "\"false should not be true\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_equals", + "test": "basic assert_array_equals test", + "args": [ + "[1, NaN]", + "[1, NaN]", + "\"[1, NaN] is equal to [1, NaN]\"" + ], + "status": 0 + }, + { + "assert_name": "assert_array_equals", + "test": "assert_array_equals with first param undefined", + "args": [ + "undefined", + "[1]", + "\"undefined equals [1]?\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_equals", + "test": "assert_array_equals with first param true", + "args": [ + "true", + "[1]", + "\"true equals [1]?\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_equals", + "test": "assert_array_equals with first param false", + "args": [ + "false", + "[1]", + "\"false equals [1]?\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_equals", + "test": "assert_array_equals with first param null", + "args": [ + "null", + "[1]", + "\"null equals [1]?\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_equals", + "test": "assert_array_equals with first param 1", + "args": [ + "1", + "[1]", + "\"1 equals [1]?\"" + ], + "status": 1 + }, + { + "assert_name": "assert_array_approx_equals", + "test": "basic assert_array_approx_equals test", + "args": [ + "[10, 11]", + "[11, 10]", + "1", + "\"[10, 11] is approximately (+/- 1) [11, 10]\"" + ], + "status": 0 + }, + { + "assert_name": "assert_approx_equals", + "test": "basic assert_approx_equals test", + "args": [ + "10", + "11", + "1", + "\"10 is approximately (+/- 1) 11\"" + ], + "status": 0 + }, + { + "assert_name": "assert_approx_equals", + "test": "basic assert_less_than test", + "args": [ + "10", + "11", + "1", + "\"10 is approximately (+/- 1) 11\"" + ], + "status": 0 + }, + { + "assert_name": "assert_greater_than", + "test": "assert_greater_than expected to fail", + "args": [ + "10", + "11", + "\"10 is not greater than 11\"" + ], + "status": 1 + }, + { + "assert_name": "assert_greater_than_equal", + "test": "basic assert_greater_than_equal test", + "args": [ + "10", + "10", + "\"10 is greater than or equal to 10\"" + ], + "status": 0 + }, + { + "assert_name": "assert_greater_than_equal", + "test": "assert_less_than_equal expected to fail", + "args": [ + "\"10\"", + "10", + "\"'10' is not a number\"" + ], + "status": 1 + }, + { + "assert_name": "assert_own_property", + "test": "test for assert[_not]_own_property and insert_inherits", + "args": [ + "object \"[object Object]\"", + "\"a\"" + ], + "status": 0 + }, + { + "assert_name": "assert_not_own_property", + "test": "test for assert[_not]_own_property and insert_inherits", + "args": [ + "object \"[object Object]\"", + "\"b\"", + "\"unexpected property found: \\\"b\\\"\"" + ], + "status": 0 + }, + { + "assert_name": "assert_inherits", + "test": "test for assert[_not]_own_property and insert_inherits", + "args": [ + "object \"[object Object]\"", + "\"b\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM exception", + "args": [ + "\"NOT_FOUND_ERR\"", + "function \"function () {a.removeChild(b)}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_js", + "test": "Test throw DOMException as JS exception expected to fail", + "args": [ + "function \"function DOMException() { [native code] }\"", + "function \"function () {a.removeChild(b)}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_js", + "test": "Test throw SyntaxError DOMException where JS SyntaxError expected; expected to fail", + "args": [ + "function \"function SyntaxError() { [native code] }\"", + "function \"function () {document.querySelector(\"\")}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_js", + "test": "Test throw JS SyntaxError", + "args": [ + "function \"function SyntaxError() { [native code] }\"", + "function \"function () {JSON.parse(\"{\")}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM SyntaxError", + "args": [ + "\"SyntaxError\"", + "function \"function () {document.querySelector(\"\")}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM SyntaxError from subframe", + "args": [ + "\"SyntaxError\"", + "function \"function DOMException() { [native code] }\"", + "function \"function () {ifr.contentDocument.querySelector(\"\")}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM SyntaxError from subframe with incorrect global expectation; expected to fail", + "args": [ + "\"SyntaxError\"", + "function \"function () {ifr.contentDocument.querySelector(\"\")}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM SyntaxError with incorrect expectation; expected to fail", + "args": [ + "\"SyntaxError\"", + "function \"function DOMException() { [native code] }\"", + "function \"function () {document.querySelector(\"\")}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw JS SyntaxError where SyntaxError DOMException expected; expected to fail", + "args": [ + "\"SyntaxError\"", + "function \"function () {JSON.parse(\"{\")}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test throw DOM exception expected to fail", + "args": [ + "\"NOT_FOUND_ERR\"", + "function \"function () {a.appendChild(b)}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test assert_throws_dom with ambiguous DOM-exception expected to Fail", + "args": [ + "0", + "function \"function() {throw e}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test assert_throws_dom with non-DOM-exception expected to Fail", + "args": [ + "\"TEST_ERR\"", + "function \"function() {throw e}\"" + ], + "status": 1 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test assert_throws_dom with number code value expected to Pass", + "args": [ + "12", + "function \"function() {throw e}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_throws_dom", + "test": "Test assert_throws_dom with number code value and real DOMException expected to Pass", + "args": [ + "12", + "function \"function() {throw e}\"" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "Test that defines a global and cleans it up", + "args": [ + "1", + "1" + ], + "status": 0 + }, + { + "assert_name": "assert_equals", + "test": "Test that cleanup handlers from previous test ran", + "args": [ + "undefined", + "undefined" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "Test step_func", + "args": [ + "true" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "Test async test with callback", + "args": [ + "true" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "Test async test with callback and `this` obj.", + "args": [ + "true" + ], + "status": 0 + } + ], "type": "complete" } </script>
diff --git a/resources/test/tests/functional/api-tests-2.html b/resources/test/tests/functional/api-tests-2.html index 08a2b89..ba7a08f 100644 --- a/resources/test/tests/functional/api-tests-2.html +++ b/resources/test/tests/functional/api-tests-2.html
@@ -40,6 +40,24 @@ "message": null } ], + "summarized_asserts": [ + { + "assert_name": "assert_true", + "test": "Test defined before onload", + "args": [ + "true" + ], + "status": 0 + }, + { + "assert_name": "assert_true", + "test": "Test defined after onload", + "args": [ + "true" + ], + "status": 0 + } + ], "type": "complete" } </script>