subunit

Merge lp:~jml/subunit/masked-errors into lp:~subunit/subunit/trunk

  1. masked-errors
  2. Merge into trunk
Proposed by Jonathan Lange
Status: Rejected
Rejected by: Jonathan Lange
Proposed branch: lp:~jml/subunit/masked-errors
Merge into: lp:~subunit/subunit/trunk
Diff against target: 163 lines (+55/-16)
1 file modified
python/subunit/tests/test_test_protocol.py (+55/-16)
To merge this branch: bzr merge lp:~jml/subunit/masked-errors
Reviewer Review Type Date Requested Status
Subunit Developers Pending
Review via email: mp+37725@code.launchpad.net

Description of the change

This branch is a work in progress. It adds a test that reproduces an error that we are seeing on Launchpad.

At the guts of it, if there's a "test:" that isn't followed up by some kind of matching result command (e.g. "failure:", "success:"), then every single other test in the stream is ignored.

I've stopped at adding the test, since this particular behaviour of subunit's might be construed as a feature.

To post a comment you must log in.
Revision history for this message
Robert Collins (lifeless) wrote :

yes, this is a feature.

It would be nice to:
 - have a way to have cake and eat it
 - have a way to recover and get both the failure and the real tests
thereafter. Robustly.

Revision history for this message
Jonathan Lange (jml) wrote :

> - have a way to recover and get both the failure and the real tests
> thereafter. Robustly.

What would this look like, I wonder.

Unmerged revisions

136. By Jonathan Lange

Simple test that reproduces the error we're seeing in Launchpad.

135. By Jonathan Lange

Clean up some pyflakes

134. By Jonathan Lange

Whitespace

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'python/subunit/tests/test_test_protocol.py'
2--- python/subunit/tests/test_test_protocol.py 2010-06-24 10:02:35 +0000
3+++ python/subunit/tests/test_test_protocol.py 2010-10-06 11:11:09 +0000
4@@ -6,7 +6,7 @@
5 # license at the users choice. A copy of both licenses are available in the
6 # project source as Apache-2.0 and BSD. You may not use this file except in
7 # compliance with one of these two licences.
8-#
9+#
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13@@ -18,10 +18,11 @@
14 import unittest
15 from StringIO import StringIO
16 import os
17-import sys
18
19+from testtools import TestCase
20 from testtools.content import Content, TracebackContent
21 from testtools.content_type import ContentType
22+from testtools.matchers import Equals
23 from testtools.tests.helpers import (
24 Python26TestResult,
25 Python27TestResult,
26@@ -61,7 +62,6 @@
27 pipe = StringIO("test old mcdonald\n"
28 "success old mcdonald\n")
29 protocol.readFrom(pipe)
30- mcdonald = subunit.RemotedTestCase("old mcdonald")
31 self.assertEqual(client.testsRun, 1)
32 self.assertEqual(pipe.getvalue(), out.getvalue())
33
34@@ -74,7 +74,7 @@
35 protocol.readFrom(pipe)
36 self.assertEqual(client.testsRun, 0)
37 self.assertEqual("", out.getvalue())
38-
39+
40
41 class TestTestProtocolServerPipe(unittest.TestCase):
42
43@@ -90,7 +90,6 @@
44 "test an error\n"
45 "error an error\n")
46 protocol.readFrom(pipe)
47- mcdonald = subunit.RemotedTestCase("old mcdonald")
48 bing = subunit.RemotedTestCase("bing crosby")
49 an_error = subunit.RemotedTestCase("an error")
50 self.assertEqual(client.errors,
51@@ -311,7 +310,7 @@
52 self.protocol.lineReceived("%s old mcdonald %s" % (outcome, opening))
53 self.protocol.lostConnection()
54 failure = subunit.RemoteError(
55- u"lost connection during %s report of test 'old mcdonald'" %
56+ u"lost connection during %s report of test 'old mcdonald'" %
57 outcome)
58 self.assertEqual([
59 ('startTest', self.test),
60@@ -388,9 +387,10 @@
61 subunit.details.MultipartDetailsParser))
62
63
64-class TestTestProtocolServerAddError(unittest.TestCase):
65+class TestTestProtocolServerAddError(TestCase):
66
67 def setUp(self):
68+ super(TestTestProtocolServerAddError, self).setUp()
69 self.client = ExtendedTestResult()
70 self.protocol = subunit.TestProtocolServer(self.client)
71 self.protocol.lineReceived("test mcdonalds farm\n")
72@@ -442,6 +442,51 @@
73 def test_error_colon_quoted_bracket(self):
74 self.error_quoted_bracket("error:")
75
76+ def test_corrupted_stream(self):
77+ # Reproduce an error that occured using the Launchpad test suite.
78+ # Something in the subunit stream caused later errors to be ignored.
79+ client = ExtendedTestResult()
80+ protocol = subunit.TestProtocolServer(client)
81+ kaboom_error = """\
82+Traceback (most recent call last):
83+ File "layers.py", line 610, in _check_and_reset
84+ "shutdown: " + str(e)
85+KaboomError: Something completely failed to handle an error
86+"""
87+ subsequent_error = """\
88+Traceback (most recent call last):
89+ File "/usr/lib/python2.6/doctest.py", line 2152, in runTest
90+ raise self.failureException(self.format_failure(new.getvalue()))
91+AssertionError: Normal failure that ought to be reported
92+"""
93+ subunit_stream = """\
94+test: kaboom
95+%s
96+test: subsequent-failure
97+failure: subsequent-failure [
98+%s
99+]
100+""" % (kaboom_error, subsequent_error)
101+ map(protocol.lineReceived, subunit_stream.splitlines(True))
102+ # XXX: I don't know what we actually *want* here. The events below
103+ # are my best guess.
104+ kaboom = subunit.RemotedTestCase('kaboom')
105+ failure = subunit.RemotedTestCase('subsequent-failure')
106+ expected_events = [
107+ ('startTest', kaboom),
108+ # Since we have started to receive more subunit output, we should
109+ # mark 'kaboom' as being erroneous, somehow. Not sure what the
110+ # exception should be.
111+ ('addError', kaboom, ''),
112+ ('stopTest', kaboom),
113+ ('startTest', failure),
114+ ('addFailure', failure,
115+ Content(ContentType('text', 'x-traceback', {'charset': 'utf8'}),
116+ lambda: [subsequent_error])),
117+ ('stopTest', failure),
118+ ]
119+ self.assertThat(client._events, Equals(expected_events))
120+
121
122 class TestTestProtocolServerAddFailure(unittest.TestCase):
123
124@@ -681,12 +726,6 @@
125 ], self.client._events)
126
127 def test_simple_success(self):
128- self.simple_success_keyword("failure")
129-
130- def test_simple_success_colon(self):
131- self.simple_success_keyword("failure:")
132-
133- def test_simple_success(self):
134 self.simple_success_keyword("successful")
135
136 def test_simple_success_colon(self):
137@@ -946,7 +985,7 @@
138
139
140 def test_construct(self):
141- test = self.SampleIsolatedTestCase("test_sets_global_state")
142+ self.SampleIsolatedTestCase("test_sets_global_state")
143
144 def test_run(self):
145 result = unittest.TestResult()
146@@ -982,7 +1021,7 @@
147
148
149 def test_construct(self):
150- suite = subunit.IsolatedTestSuite()
151+ subunit.IsolatedTestSuite()
152
153 def test_run(self):
154 result = unittest.TestResult()
155@@ -1117,7 +1156,7 @@
156 self.assertEqual(
157 self.io.getvalue(),
158 'skip: %s [\nHas it really?\n]\n' % self.test.id())
159-
160+
161 def test_add_skip_details(self):
162 """Test addSkip on a TestProtocolClient with details."""
163 details = {'reason':Content(

Subscribers

People subscribed via source and target branches