subunit

Merge lp:~jml/subunit/masked-errors into lp:~subunit/subunit/trunk

  1. masked-errors
  2. Merge into trunk
Proposed by Jonathan Lange
Status: Rejected
Rejected by: Jonathan Lange
Proposed branch: lp:~jml/subunit/masked-errors
Merge into: lp:~subunit/subunit/trunk
Diff against target: 163 lines (+55/-16)
1 file modified
python/subunit/tests/test_test_protocol.py (+55/-16)
To merge this branch: bzr merge lp:~jml/subunit/masked-errors
Reviewer Review Type Date Requested Status
Subunit Developers Pending
Review via email: mp+37725@code.launchpad.net

Description of the change

This branch is a work in progress. It adds a test that reproduces an error that we are seeing on Launchpad.

At the guts of it, if there's a "test:" that isn't followed up by some kind of matching result command (e.g. "failure:", "success:"), then every single other test in the stream is ignored.

I've stopped at adding the test, since this particular behaviour of subunit's might be construed as a feature.

To post a comment you must log in.
Revision history for this message
Robert Collins (lifeless) wrote :

yes, this is a feature.

It would be nice to:
 - have a way to have cake and eat it
 - have a way to recover and get both the failure and the real tests
thereafter. Robustly.

Revision history for this message
Jonathan Lange (jml) wrote :

> - have a way to recover and get both the failure and the real tests
> thereafter. Robustly.

What would this look like, I wonder.

Unmerged revisions

136. By Jonathan Lange

Simple test that reproduces the error we're seeing in Launchpad.

135. By Jonathan Lange

Clean up some pyflakes

134. By Jonathan Lange

Whitespace

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'python/subunit/tests/test_test_protocol.py'
--- python/subunit/tests/test_test_protocol.py 2010-06-24 10:02:35 +0000
+++ python/subunit/tests/test_test_protocol.py 2010-10-06 11:11:09 +0000
@@ -6,7 +6,7 @@
6# license at the users choice. A copy of both licenses are available in the6# license at the users choice. A copy of both licenses are available in the
7# project source as Apache-2.0 and BSD. You may not use this file except in7# project source as Apache-2.0 and BSD. You may not use this file except in
8# compliance with one of these two licences.8# compliance with one of these two licences.
9# 9#
10# Unless required by applicable law or agreed to in writing, software10# Unless required by applicable law or agreed to in writing, software
11# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT11# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -18,10 +18,11 @@
18import unittest18import unittest
19from StringIO import StringIO19from StringIO import StringIO
20import os20import os
21import sys
2221
22from testtools import TestCase
23from testtools.content import Content, TracebackContent23from testtools.content import Content, TracebackContent
24from testtools.content_type import ContentType24from testtools.content_type import ContentType
25from testtools.matchers import Equals
25from testtools.tests.helpers import (26from testtools.tests.helpers import (
26 Python26TestResult,27 Python26TestResult,
27 Python27TestResult,28 Python27TestResult,
@@ -61,7 +62,6 @@
61 pipe = StringIO("test old mcdonald\n"62 pipe = StringIO("test old mcdonald\n"
62 "success old mcdonald\n")63 "success old mcdonald\n")
63 protocol.readFrom(pipe)64 protocol.readFrom(pipe)
64 mcdonald = subunit.RemotedTestCase("old mcdonald")
65 self.assertEqual(client.testsRun, 1)65 self.assertEqual(client.testsRun, 1)
66 self.assertEqual(pipe.getvalue(), out.getvalue())66 self.assertEqual(pipe.getvalue(), out.getvalue())
6767
@@ -74,7 +74,7 @@
74 protocol.readFrom(pipe)74 protocol.readFrom(pipe)
75 self.assertEqual(client.testsRun, 0)75 self.assertEqual(client.testsRun, 0)
76 self.assertEqual("", out.getvalue())76 self.assertEqual("", out.getvalue())
77 77
7878
79class TestTestProtocolServerPipe(unittest.TestCase):79class TestTestProtocolServerPipe(unittest.TestCase):
8080
@@ -90,7 +90,6 @@
90 "test an error\n"90 "test an error\n"
91 "error an error\n")91 "error an error\n")
92 protocol.readFrom(pipe)92 protocol.readFrom(pipe)
93 mcdonald = subunit.RemotedTestCase("old mcdonald")
94 bing = subunit.RemotedTestCase("bing crosby")93 bing = subunit.RemotedTestCase("bing crosby")
95 an_error = subunit.RemotedTestCase("an error")94 an_error = subunit.RemotedTestCase("an error")
96 self.assertEqual(client.errors,95 self.assertEqual(client.errors,
@@ -311,7 +310,7 @@
311 self.protocol.lineReceived("%s old mcdonald %s" % (outcome, opening))310 self.protocol.lineReceived("%s old mcdonald %s" % (outcome, opening))
312 self.protocol.lostConnection()311 self.protocol.lostConnection()
313 failure = subunit.RemoteError(312 failure = subunit.RemoteError(
314 u"lost connection during %s report of test 'old mcdonald'" % 313 u"lost connection during %s report of test 'old mcdonald'" %
315 outcome)314 outcome)
316 self.assertEqual([315 self.assertEqual([
317 ('startTest', self.test),316 ('startTest', self.test),
@@ -388,9 +387,10 @@
388 subunit.details.MultipartDetailsParser))387 subunit.details.MultipartDetailsParser))
389388
390389
391class TestTestProtocolServerAddError(unittest.TestCase):390class TestTestProtocolServerAddError(TestCase):
392391
393 def setUp(self):392 def setUp(self):
393 super(TestTestProtocolServerAddError, self).setUp()
394 self.client = ExtendedTestResult()394 self.client = ExtendedTestResult()
395 self.protocol = subunit.TestProtocolServer(self.client)395 self.protocol = subunit.TestProtocolServer(self.client)
396 self.protocol.lineReceived("test mcdonalds farm\n")396 self.protocol.lineReceived("test mcdonalds farm\n")
@@ -442,6 +442,51 @@
442 def test_error_colon_quoted_bracket(self):442 def test_error_colon_quoted_bracket(self):
443 self.error_quoted_bracket("error:")443 self.error_quoted_bracket("error:")
444444
445 def test_corrupted_stream(self):
446 # Reproduce an error that occured using the Launchpad test suite.
447 # Something in the subunit stream caused later errors to be ignored.
448 client = ExtendedTestResult()
449 protocol = subunit.TestProtocolServer(client)
450 kaboom_error = """\
451Traceback (most recent call last):
452 File "layers.py", line 610, in _check_and_reset
453 "shutdown: " + str(e)
454KaboomError: Something completely failed to handle an error
455"""
456 subsequent_error = """\
457Traceback (most recent call last):
458 File "/usr/lib/python2.6/doctest.py", line 2152, in runTest
459 raise self.failureException(self.format_failure(new.getvalue()))
460AssertionError: Normal failure that ought to be reported
461"""
462 subunit_stream = """\
463test: kaboom
464%s
465test: subsequent-failure
466failure: subsequent-failure [
467%s
468]
469""" % (kaboom_error, subsequent_error)
470 map(protocol.lineReceived, subunit_stream.splitlines(True))
471 # XXX: I don't know what we actually *want* here. The events below
472 # are my best guess.
473 kaboom = subunit.RemotedTestCase('kaboom')
474 failure = subunit.RemotedTestCase('subsequent-failure')
475 expected_events = [
476 ('startTest', kaboom),
477 # Since we have started to receive more subunit output, we should
478 # mark 'kaboom' as being erroneous, somehow. Not sure what the
479 # exception should be.
480 ('addError', kaboom, ''),
481 ('stopTest', kaboom),
482 ('startTest', failure),
483 ('addFailure', failure,
484 Content(ContentType('text', 'x-traceback', {'charset': 'utf8'}),
485 lambda: [subsequent_error])),
486 ('stopTest', failure),
487 ]
488 self.assertThat(client._events, Equals(expected_events))
489
445490
446class TestTestProtocolServerAddFailure(unittest.TestCase):491class TestTestProtocolServerAddFailure(unittest.TestCase):
447492
@@ -681,12 +726,6 @@
681 ], self.client._events)726 ], self.client._events)
682727
683 def test_simple_success(self):728 def test_simple_success(self):
684 self.simple_success_keyword("failure")
685
686 def test_simple_success_colon(self):
687 self.simple_success_keyword("failure:")
688
689 def test_simple_success(self):
690 self.simple_success_keyword("successful")729 self.simple_success_keyword("successful")
691730
692 def test_simple_success_colon(self):731 def test_simple_success_colon(self):
@@ -946,7 +985,7 @@
946985
947986
948 def test_construct(self):987 def test_construct(self):
949 test = self.SampleIsolatedTestCase("test_sets_global_state")988 self.SampleIsolatedTestCase("test_sets_global_state")
950989
951 def test_run(self):990 def test_run(self):
952 result = unittest.TestResult()991 result = unittest.TestResult()
@@ -982,7 +1021,7 @@
9821021
9831022
984 def test_construct(self):1023 def test_construct(self):
985 suite = subunit.IsolatedTestSuite()1024 subunit.IsolatedTestSuite()
9861025
987 def test_run(self):1026 def test_run(self):
988 result = unittest.TestResult()1027 result = unittest.TestResult()
@@ -1117,7 +1156,7 @@
1117 self.assertEqual(1156 self.assertEqual(
1118 self.io.getvalue(),1157 self.io.getvalue(),
1119 'skip: %s [\nHas it really?\n]\n' % self.test.id())1158 'skip: %s [\nHas it really?\n]\n' % self.test.id())
1120 1159
1121 def test_add_skip_details(self):1160 def test_add_skip_details(self):
1122 """Test addSkip on a TestProtocolClient with details."""1161 """Test addSkip on a TestProtocolClient with details."""
1123 details = {'reason':Content(1162 details = {'reason':Content(

Subscribers

People subscribed via source and target branches