Merge lp:~jml/launchpad/split-mail-and-summary into lp:launchpad
- split-mail-and-summary
- Merge into devel
Proposed by
Jonathan Lange
| Status: | Merged | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Approved by: | Robert Collins | ||||||||||||
| Approved revision: | no longer in the source branch. | ||||||||||||
| Merged at revision: | 11596 | ||||||||||||
| Proposed branch: | lp:~jml/launchpad/split-mail-and-summary | ||||||||||||
| Merge into: | lp:launchpad | ||||||||||||
| Diff against target: |
551 lines (+259/-46) 3 files modified
lib/devscripts/ec2test/remote.py (+87/-15) lib/devscripts/ec2test/tests/test_remote.py (+168/-30) lib/lp/services/job/tests/test_runner.py (+4/-1) |
||||||||||||
| To merge this branch: | bzr merge lp:~jml/launchpad/split-mail-and-summary | ||||||||||||
| Related bugs: |
|
| Reviewer | Review Type | Date Requested | Status |
|---|---|---|---|
| Robert Collins (community) | Approve | ||
|
Review via email:
|
|||
Commit message
Improve the email sent by ec2 test.
Description of the change
Hello,
This branch changes the email sent by ec2 test. Rather than being a mere lump of stuff that we have collected as summary, it's a custom built message derived from the test results.
Examples:
* http://
* http://
Implementation-
The details:
* If the test child process exits with a non-zero code and we have no test results, we raise an error, since it's likely to be something weird.
* Included in the diff are changes from a merge proposal that I should have applied before landing my previous branch. Mostly stylistic.
* Some tests were incorrect. They used a dictionary to fake an email message, but had the subject stored under the wrong key ("Subject:" rather than "Subject").
I look forward to your review.
jml
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
| 1 | === modified file 'lib/devscripts/ec2test/remote.py' | |||
| 2 | --- lib/devscripts/ec2test/remote.py 2010-08-27 14:27:22 +0000 | |||
| 3 | +++ lib/devscripts/ec2test/remote.py 2010-09-22 13:36:00 +0000 | |||
| 4 | @@ -28,6 +28,7 @@ | |||
| 5 | 28 | import optparse | 28 | import optparse |
| 6 | 29 | import os | 29 | import os |
| 7 | 30 | import pickle | 30 | import pickle |
| 8 | 31 | from StringIO import StringIO | ||
| 9 | 31 | import subprocess | 32 | import subprocess |
| 10 | 32 | import sys | 33 | import sys |
| 11 | 33 | import tempfile | 34 | import tempfile |
| 12 | @@ -49,6 +50,15 @@ | |||
| 13 | 49 | import subunit | 50 | import subunit |
| 14 | 50 | 51 | ||
| 15 | 51 | 52 | ||
| 16 | 53 | class NonZeroExitCode(Exception): | ||
| 17 | 54 | """Raised when the child process exits with a non-zero exit code.""" | ||
| 18 | 55 | |||
| 19 | 56 | def __init__(self, retcode): | ||
| 20 | 57 | super(NonZeroExitCode, self).__init__( | ||
| 21 | 58 | 'Test process died with exit code %r, but no tests failed.' | ||
| 22 | 59 | % (retcode,)) | ||
| 23 | 60 | |||
| 24 | 61 | |||
| 25 | 52 | class SummaryResult(unittest.TestResult): | 62 | class SummaryResult(unittest.TestResult): |
| 26 | 53 | """Test result object used to generate the summary.""" | 63 | """Test result object used to generate the summary.""" |
| 27 | 54 | 64 | ||
| 28 | @@ -271,12 +281,16 @@ | |||
| 29 | 271 | self._logger.prepare() | 281 | self._logger.prepare() |
| 30 | 272 | try: | 282 | try: |
| 31 | 273 | popen = self._spawn_test_process() | 283 | popen = self._spawn_test_process() |
| 34 | 274 | self._gather_test_output(popen.stdout, self._logger) | 284 | result = self._gather_test_output(popen.stdout, self._logger) |
| 35 | 275 | exit_status = popen.wait() | 285 | retcode = popen.wait() |
| 36 | 286 | # The process could have an error not indicated by an actual test | ||
| 37 | 287 | # result nor by a raised exception | ||
| 38 | 288 | if result.wasSuccessful() and retcode: | ||
| 39 | 289 | raise NonZeroExitCode(retcode) | ||
| 40 | 276 | except: | 290 | except: |
| 41 | 277 | self._logger.error_in_testrunner(sys.exc_info()) | 291 | self._logger.error_in_testrunner(sys.exc_info()) |
| 42 | 278 | else: | 292 | else: |
| 44 | 279 | self._logger.got_result(not exit_status) | 293 | self._logger.got_result(result) |
| 45 | 280 | 294 | ||
| 46 | 281 | def _gather_test_output(self, input_stream, logger): | 295 | def _gather_test_output(self, input_stream, logger): |
| 47 | 282 | """Write the testrunner output to the logs.""" | 296 | """Write the testrunner output to the logs.""" |
| 48 | @@ -287,6 +301,7 @@ | |||
| 49 | 287 | subunit_server.lineReceived(line) | 301 | subunit_server.lineReceived(line) |
| 50 | 288 | logger.got_line(line) | 302 | logger.got_line(line) |
| 51 | 289 | summary_stream.flush() | 303 | summary_stream.flush() |
| 52 | 304 | return result | ||
| 53 | 290 | 305 | ||
| 54 | 291 | 306 | ||
| 55 | 292 | class Request: | 307 | class Request: |
| 56 | @@ -327,6 +342,57 @@ | |||
| 57 | 327 | """Actually send 'message'.""" | 342 | """Actually send 'message'.""" |
| 58 | 328 | self._smtp_connection.send_email(message) | 343 | self._smtp_connection.send_email(message) |
| 59 | 329 | 344 | ||
| 60 | 345 | def _format_test_list(self, header, tests): | ||
| 61 | 346 | if not tests: | ||
| 62 | 347 | return [] | ||
| 63 | 348 | tests = [' ' + test.id() for test, error in tests] | ||
| 64 | 349 | return [header, '-' * len(header)] + tests + [''] | ||
| 65 | 350 | |||
| 66 | 351 | def format_result(self, result, start_time, end_time): | ||
| 67 | 352 | duration = end_time - start_time | ||
| 68 | 353 | output = [ | ||
| 69 | 354 | 'Tests started at approximately %s' % start_time, | ||
| 70 | 355 | ] | ||
| 71 | 356 | source = self.get_source_details() | ||
| 72 | 357 | if source: | ||
| 73 | 358 | output.append('Source: %s r%s' % source) | ||
| 74 | 359 | target = self.get_target_details() | ||
| 75 | 360 | if target: | ||
| 76 | 361 | output.append('Target: %s r%s' % target) | ||
| 77 | 362 | output.extend([ | ||
| 78 | 363 | '', | ||
| 79 | 364 | '%s tests run in %s, %s failures, %s errors' % ( | ||
| 80 | 365 | result.testsRun, duration, len(result.failures), | ||
| 81 | 366 | len(result.errors)), | ||
| 82 | 367 | '', | ||
| 83 | 368 | ]) | ||
| 84 | 369 | |||
| 85 | 370 | bad_tests = ( | ||
| 86 | 371 | self._format_test_list('Failing tests', result.failures) + | ||
| 87 | 372 | self._format_test_list('Tests with errors', result.errors)) | ||
| 88 | 373 | output.extend(bad_tests) | ||
| 89 | 374 | |||
| 90 | 375 | if bad_tests: | ||
| 91 | 376 | full_error_stream = StringIO() | ||
| 92 | 377 | copy_result = SummaryResult(full_error_stream) | ||
| 93 | 378 | for test, error in result.failures: | ||
| 94 | 379 | full_error_stream.write( | ||
| 95 | 380 | copy_result._formatError('FAILURE', test, error)) | ||
| 96 | 381 | for test, error in result.errors: | ||
| 97 | 382 | full_error_stream.write( | ||
| 98 | 383 | copy_result._formatError('ERROR', test, error)) | ||
| 99 | 384 | output.append(full_error_stream.getvalue()) | ||
| 100 | 385 | |||
| 101 | 386 | subject = self._get_pqm_subject() | ||
| 102 | 387 | if subject: | ||
| 103 | 388 | if result.wasSuccessful(): | ||
| 104 | 389 | output.append('SUBMITTED TO PQM:') | ||
| 105 | 390 | else: | ||
| 106 | 391 | output.append('**NOT** submitted to PQM:') | ||
| 107 | 392 | output.extend([subject, '']) | ||
| 108 | 393 | output.extend(['(See the attached file for the complete log)', '']) | ||
| 109 | 394 | return '\n'.join(output) | ||
| 110 | 395 | |||
| 111 | 330 | def get_target_details(self): | 396 | def get_target_details(self): |
| 112 | 331 | """Return (branch_url, revno) for trunk.""" | 397 | """Return (branch_url, revno) for trunk.""" |
| 113 | 332 | branch = bzrlib.branch.Branch.open(self._local_branch_path) | 398 | branch = bzrlib.branch.Branch.open(self._local_branch_path) |
| 114 | @@ -449,12 +515,15 @@ | |||
| 115 | 449 | continue | 515 | continue |
| 116 | 450 | yield name, branch.get_parent(), branch.revno() | 516 | yield name, branch.get_parent(), branch.revno() |
| 117 | 451 | 517 | ||
| 118 | 518 | def _get_pqm_subject(self): | ||
| 119 | 519 | if not self._pqm_message: | ||
| 120 | 520 | return | ||
| 121 | 521 | return self._pqm_message.get('Subject') | ||
| 122 | 522 | |||
| 123 | 452 | def submit_to_pqm(self, successful): | 523 | def submit_to_pqm(self, successful): |
| 124 | 453 | """Submit this request to PQM, if successful & configured to do so.""" | 524 | """Submit this request to PQM, if successful & configured to do so.""" |
| 129 | 454 | if not self._pqm_message: | 525 | subject = self._get_pqm_subject() |
| 130 | 455 | return | 526 | if subject and successful: |
| 127 | 456 | subject = self._pqm_message.get('Subject') | ||
| 128 | 457 | if successful: | ||
| 131 | 458 | self._send_email(self._pqm_message) | 527 | self._send_email(self._pqm_message) |
| 132 | 459 | return subject | 528 | return subject |
| 133 | 460 | 529 | ||
| 134 | @@ -491,6 +560,9 @@ | |||
| 135 | 491 | self._index_filename = index_filename | 560 | self._index_filename = index_filename |
| 136 | 492 | self._request = request | 561 | self._request = request |
| 137 | 493 | self._echo_to_stdout = echo_to_stdout | 562 | self._echo_to_stdout = echo_to_stdout |
| 138 | 563 | # Actually set by prepare(), but setting to a dummy value to make | ||
| 139 | 564 | # testing easier. | ||
| 140 | 565 | self._start_time = datetime.datetime.utcnow() | ||
| 141 | 494 | 566 | ||
| 142 | 495 | @classmethod | 567 | @classmethod |
| 143 | 496 | def make_in_directory(cls, www_dir, request, echo_to_stdout): | 568 | def make_in_directory(cls, www_dir, request, echo_to_stdout): |
| 144 | @@ -558,16 +630,16 @@ | |||
| 145 | 558 | if e.errno == errno.ENOENT: | 630 | if e.errno == errno.ENOENT: |
| 146 | 559 | return '' | 631 | return '' |
| 147 | 560 | 632 | ||
| 149 | 561 | def got_result(self, successful): | 633 | def got_result(self, result): |
| 150 | 562 | """The tests are done and the results are known.""" | 634 | """The tests are done and the results are known.""" |
| 151 | 635 | self._end_time = datetime.datetime.utcnow() | ||
| 152 | 636 | successful = result.wasSuccessful() | ||
| 153 | 563 | self._handle_pqm_submission(successful) | 637 | self._handle_pqm_submission(successful) |
| 154 | 564 | if self._request.wants_email: | 638 | if self._request.wants_email: |
| 159 | 565 | self._write_to_filename( | 639 | email_text = self._request.format_result( |
| 160 | 566 | self._summary_filename, | 640 | result, self._start_time, self._end_time) |
| 157 | 567 | '\n(See the attached file for the complete log)\n') | ||
| 158 | 568 | summary = self.get_summary_contents() | ||
| 161 | 569 | full_log_gz = gzip_data(self.get_full_log_contents()) | 641 | full_log_gz = gzip_data(self.get_full_log_contents()) |
| 163 | 570 | self._request.send_report_email(successful, summary, full_log_gz) | 642 | self._request.send_report_email(successful, email_text, full_log_gz) |
| 164 | 571 | 643 | ||
| 165 | 572 | def _handle_pqm_submission(self, successful): | 644 | def _handle_pqm_submission(self, successful): |
| 166 | 573 | subject = self._request.submit_to_pqm(successful) | 645 | subject = self._request.submit_to_pqm(successful) |
| 167 | @@ -614,9 +686,9 @@ | |||
| 168 | 614 | return self._write_to_filename( | 686 | return self._write_to_filename( |
| 169 | 615 | self._index_filename, textwrap.dedent(html)) | 687 | self._index_filename, textwrap.dedent(html)) |
| 170 | 616 | 688 | ||
| 171 | 689 | self._start_time = datetime.datetime.utcnow() | ||
| 172 | 617 | msg = 'Tests started at approximately %(now)s UTC' % { | 690 | msg = 'Tests started at approximately %(now)s UTC' % { |
| 175 | 618 | 'now': datetime.datetime.utcnow().strftime( | 691 | 'now': self._start_time.strftime('%a, %d %b %Y %H:%M:%S')} |
| 174 | 619 | '%a, %d %b %Y %H:%M:%S')} | ||
| 176 | 620 | add_to_html('''\ | 692 | add_to_html('''\ |
| 177 | 621 | <html> | 693 | <html> |
| 178 | 622 | <head> | 694 | <head> |
| 179 | 623 | 695 | ||
| 180 | === modified file 'lib/devscripts/ec2test/tests/test_remote.py' | |||
| 181 | --- lib/devscripts/ec2test/tests/test_remote.py 2010-08-27 14:27:22 +0000 | |||
| 182 | +++ lib/devscripts/ec2test/tests/test_remote.py 2010-09-22 13:36:00 +0000 | |||
| 183 | @@ -5,6 +5,8 @@ | |||
| 184 | 5 | 5 | ||
| 185 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
| 186 | 7 | 7 | ||
| 187 | 8 | from datetime import datetime, timedelta | ||
| 188 | 9 | import doctest | ||
| 189 | 8 | from email.mime.application import MIMEApplication | 10 | from email.mime.application import MIMEApplication |
| 190 | 9 | from email.mime.text import MIMEText | 11 | from email.mime.text import MIMEText |
| 191 | 10 | import gzip | 12 | import gzip |
| 192 | @@ -21,9 +23,10 @@ | |||
| 193 | 21 | from bzrlib.config import GlobalConfig | 23 | from bzrlib.config import GlobalConfig |
| 194 | 22 | from bzrlib.tests import TestCaseWithTransport | 24 | from bzrlib.tests import TestCaseWithTransport |
| 195 | 23 | 25 | ||
| 197 | 24 | from testtools import TestCase | 26 | from testtools import TestCase, TestResult |
| 198 | 25 | from testtools.content import Content | 27 | from testtools.content import Content |
| 199 | 26 | from testtools.content_type import ContentType | 28 | from testtools.content_type import ContentType |
| 200 | 29 | from testtools.matchers import DocTestMatches | ||
| 201 | 27 | 30 | ||
| 202 | 28 | from devscripts.ec2test.remote import ( | 31 | from devscripts.ec2test.remote import ( |
| 203 | 29 | EC2Runner, | 32 | EC2Runner, |
| 204 | @@ -75,7 +78,8 @@ | |||
| 205 | 75 | """Make a request to test, specifying only things we care about. | 78 | """Make a request to test, specifying only things we care about. |
| 206 | 76 | 79 | ||
| 207 | 77 | Note that the returned request object will not ever send email, but | 80 | Note that the returned request object will not ever send email, but |
| 209 | 78 | will instead log "sent" emails to `request.emails_sent`. | 81 | will instead append "sent" emails to the list provided here as |
| 210 | 82 | 'emails_sent'. | ||
| 211 | 79 | """ | 83 | """ |
| 212 | 80 | if trunk is None: | 84 | if trunk is None: |
| 213 | 81 | trunk = self.make_trunk() | 85 | trunk = self.make_trunk() |
| 214 | @@ -109,8 +113,6 @@ | |||
| 215 | 109 | def make_tester(self, logger=None, test_directory=None, test_options=()): | 113 | def make_tester(self, logger=None, test_directory=None, test_options=()): |
| 216 | 110 | if not logger: | 114 | if not logger: |
| 217 | 111 | logger = self.make_logger() | 115 | logger = self.make_logger() |
| 218 | 112 | if not test_directory: | ||
| 219 | 113 | test_directory = 'unspecified-test-directory' | ||
| 220 | 114 | return LaunchpadTester(logger, test_directory, test_options) | 116 | return LaunchpadTester(logger, test_directory, test_options) |
| 221 | 115 | 117 | ||
| 222 | 116 | def make_logger(self, request=None, echo_to_stdout=False): | 118 | def make_logger(self, request=None, echo_to_stdout=False): |
| 223 | @@ -472,14 +474,14 @@ | |||
| 224 | 472 | def test_submit_to_pqm_unsuccessful(self): | 474 | def test_submit_to_pqm_unsuccessful(self): |
| 225 | 473 | # submit_to_pqm returns the subject of the PQM mail even if it's | 475 | # submit_to_pqm returns the subject of the PQM mail even if it's |
| 226 | 474 | # handling a failed test run. | 476 | # handling a failed test run. |
| 228 | 475 | message = {'Subject:': 'My PQM message'} | 477 | message = {'Subject': 'My PQM message'} |
| 229 | 476 | req = self.make_request(pqm_message=message) | 478 | req = self.make_request(pqm_message=message) |
| 230 | 477 | subject = req.submit_to_pqm(successful=False) | 479 | subject = req.submit_to_pqm(successful=False) |
| 231 | 478 | self.assertIs(message.get('Subject'), subject) | 480 | self.assertIs(message.get('Subject'), subject) |
| 232 | 479 | 481 | ||
| 233 | 480 | def test_submit_to_pqm_unsuccessful_no_email(self): | 482 | def test_submit_to_pqm_unsuccessful_no_email(self): |
| 234 | 481 | # submit_to_pqm doesn't send any email if the run was unsuccessful. | 483 | # submit_to_pqm doesn't send any email if the run was unsuccessful. |
| 236 | 482 | message = {'Subject:': 'My PQM message'} | 484 | message = {'Subject': 'My PQM message'} |
| 237 | 483 | log = [] | 485 | log = [] |
| 238 | 484 | req = self.make_request(pqm_message=message, emails_sent=log) | 486 | req = self.make_request(pqm_message=message, emails_sent=log) |
| 239 | 485 | req.submit_to_pqm(successful=False) | 487 | req.submit_to_pqm(successful=False) |
| 240 | @@ -487,7 +489,7 @@ | |||
| 241 | 487 | 489 | ||
| 242 | 488 | def test_submit_to_pqm_successful(self): | 490 | def test_submit_to_pqm_successful(self): |
| 243 | 489 | # submit_to_pqm returns the subject of the PQM mail. | 491 | # submit_to_pqm returns the subject of the PQM mail. |
| 245 | 490 | message = {'Subject:': 'My PQM message'} | 492 | message = {'Subject': 'My PQM message'} |
| 246 | 491 | log = [] | 493 | log = [] |
| 247 | 492 | req = self.make_request(pqm_message=message, emails_sent=log) | 494 | req = self.make_request(pqm_message=message, emails_sent=log) |
| 248 | 493 | subject = req.submit_to_pqm(successful=True) | 495 | subject = req.submit_to_pqm(successful=True) |
| 249 | @@ -558,6 +560,124 @@ | |||
| 250 | 558 | self.assertEqual( | 560 | self.assertEqual( |
| 251 | 559 | expected_part.get_payload(), observed_part.get_payload()) | 561 | expected_part.get_payload(), observed_part.get_payload()) |
| 252 | 560 | 562 | ||
| 253 | 563 | def test_format_result_success(self): | ||
| 254 | 564 | class SomeTest(TestCase): | ||
| 255 | 565 | def test_a(self): | ||
| 256 | 566 | pass | ||
| 257 | 567 | def test_b(self): | ||
| 258 | 568 | pass | ||
| 259 | 569 | def test_c(self): | ||
| 260 | 570 | pass | ||
| 261 | 571 | test = unittest.TestSuite(map(SomeTest, ['test_' + x for x in 'abc'])) | ||
| 262 | 572 | result = TestResult() | ||
| 263 | 573 | test.run(result) | ||
| 264 | 574 | tree = self.make_trunk() | ||
| 265 | 575 | # Fake a merge, giving silly revision ids. | ||
| 266 | 576 | tree.add_pending_merge('foo', 'bar') | ||
| 267 | 577 | req = self.make_request( | ||
| 268 | 578 | branch_url='https://example.com/bzr/thing', revno=42, trunk=tree) | ||
| 269 | 579 | source_branch, source_revno = req.get_source_details() | ||
| 270 | 580 | target_branch, target_revno = req.get_target_details() | ||
| 271 | 581 | start_time = datetime.utcnow() | ||
| 272 | 582 | end_time = start_time + timedelta(hours=1) | ||
| 273 | 583 | data = { | ||
| 274 | 584 | 'source_branch': source_branch, | ||
| 275 | 585 | 'source_revno': source_revno, | ||
| 276 | 586 | 'target_branch': target_branch, | ||
| 277 | 587 | 'target_revno': target_revno, | ||
| 278 | 588 | 'start_time': str(start_time), | ||
| 279 | 589 | 'duration': str(end_time - start_time), | ||
| 280 | 590 | 'num_tests': result.testsRun, | ||
| 281 | 591 | 'num_failures': len(result.failures), | ||
| 282 | 592 | 'num_errors': len(result.errors), | ||
| 283 | 593 | } | ||
| 284 | 594 | result_text = req.format_result(result, start_time, end_time) | ||
| 285 | 595 | self.assertThat( | ||
| 286 | 596 | result_text, DocTestMatches("""\ | ||
| 287 | 597 | Tests started at approximately %(start_time)s | ||
| 288 | 598 | Source: %(source_branch)s r%(source_revno)s | ||
| 289 | 599 | Target: %(target_branch)s r%(target_revno)s | ||
| 290 | 600 | <BLANKLINE> | ||
| 291 | 601 | %(num_tests)s tests run in %(duration)s, %(num_failures)s failures, %(num_errors)s errors | ||
| 292 | 602 | <BLANKLINE> | ||
| 293 | 603 | (See the attached file for the complete log) | ||
| 294 | 604 | """ % data, doctest.REPORT_NDIFF | doctest.ELLIPSIS)) | ||
| 295 | 605 | |||
| 296 | 606 | def test_format_result_with_errors(self): | ||
| 297 | 607 | class SomeTest(TestCase): | ||
| 298 | 608 | def test_ok(self): | ||
| 299 | 609 | pass | ||
| 300 | 610 | def test_fail(self): | ||
| 301 | 611 | self.fail("oh no") | ||
| 302 | 612 | def test_error(self): | ||
| 303 | 613 | 1/0 | ||
| 304 | 614 | fail_test = SomeTest('test_fail') | ||
| 305 | 615 | error_test = SomeTest('test_error') | ||
| 306 | 616 | test = unittest.TestSuite( | ||
| 307 | 617 | [fail_test, error_test, SomeTest('test_ok')]) | ||
| 308 | 618 | result = TestResult() | ||
| 309 | 619 | test.run(result) | ||
| 310 | 620 | tree = self.make_trunk() | ||
| 311 | 621 | # Fake a merge, giving silly revision ids. | ||
| 312 | 622 | tree.add_pending_merge('foo', 'bar') | ||
| 313 | 623 | req = self.make_request( | ||
| 314 | 624 | branch_url='https://example.com/bzr/thing', revno=42, trunk=tree) | ||
| 315 | 625 | source_branch, source_revno = req.get_source_details() | ||
| 316 | 626 | target_branch, target_revno = req.get_target_details() | ||
| 317 | 627 | start_time = datetime.utcnow() | ||
| 318 | 628 | end_time = start_time + timedelta(hours=1) | ||
| 319 | 629 | data = { | ||
| 320 | 630 | 'source_branch': source_branch, | ||
| 321 | 631 | 'source_revno': source_revno, | ||
| 322 | 632 | 'target_branch': target_branch, | ||
| 323 | 633 | 'target_revno': target_revno, | ||
| 324 | 634 | 'start_time': str(start_time), | ||
| 325 | 635 | 'duration': str(end_time - start_time), | ||
| 326 | 636 | 'fail_id': fail_test.id(), | ||
| 327 | 637 | 'error_id': error_test.id(), | ||
| 328 | 638 | 'num_tests': result.testsRun, | ||
| 329 | 639 | 'num_failures': len(result.failures), | ||
| 330 | 640 | 'num_errors': len(result.errors), | ||
| 331 | 641 | } | ||
| 332 | 642 | result_text = req.format_result(result, start_time, end_time) | ||
| 333 | 643 | self.assertThat( | ||
| 334 | 644 | result_text, DocTestMatches("""\ | ||
| 335 | 645 | Tests started at approximately %(start_time)s | ||
| 336 | 646 | Source: %(source_branch)s r%(source_revno)s | ||
| 337 | 647 | Target: %(target_branch)s r%(target_revno)s | ||
| 338 | 648 | <BLANKLINE> | ||
| 339 | 649 | %(num_tests)s tests run in %(duration)s, %(num_failures)s failures, %(num_errors)s errors | ||
| 340 | 650 | <BLANKLINE> | ||
| 341 | 651 | Failing tests | ||
| 342 | 652 | ------------- | ||
| 343 | 653 | %(fail_id)s | ||
| 344 | 654 | <BLANKLINE> | ||
| 345 | 655 | Tests with errors | ||
| 346 | 656 | ----------------- | ||
| 347 | 657 | %(error_id)s | ||
| 348 | 658 | <BLANKLINE> | ||
| 349 | 659 | ====================================================================== | ||
| 350 | 660 | FAILURE: test_fail... | ||
| 351 | 661 | ---------------------------------------------------------------------- | ||
| 352 | 662 | Text attachment: traceback | ||
| 353 | 663 | ------------ | ||
| 354 | 664 | Traceback (most recent call last): | ||
| 355 | 665 | ... | ||
| 356 | 666 | ------------ | ||
| 357 | 667 | <BLANKLINE> | ||
| 358 | 668 | ====================================================================== | ||
| 359 | 669 | ERROR: test_error... | ||
| 360 | 670 | ---------------------------------------------------------------------- | ||
| 361 | 671 | Text attachment: traceback | ||
| 362 | 672 | ------------ | ||
| 363 | 673 | Traceback (most recent call last): | ||
| 364 | 674 | ... | ||
| 365 | 675 | ------------ | ||
| 366 | 676 | <BLANKLINE> | ||
| 367 | 677 | <BLANKLINE> | ||
| 368 | 678 | (See the attached file for the complete log) | ||
| 369 | 679 | """ % data, doctest.REPORT_NDIFF | doctest.ELLIPSIS)) | ||
| 370 | 680 | |||
| 371 | 561 | 681 | ||
| 372 | 562 | class TestWebTestLogger(TestCaseWithTransport, RequestHelpers): | 682 | class TestWebTestLogger(TestCaseWithTransport, RequestHelpers): |
| 373 | 563 | 683 | ||
| 374 | @@ -672,11 +792,13 @@ | |||
| 375 | 672 | False, "who-cares.pid", False, smtp_connection, emails=emails) | 792 | False, "who-cares.pid", False, smtp_connection, emails=emails) |
| 376 | 673 | 793 | ||
| 377 | 674 | def test_run(self): | 794 | def test_run(self): |
| 378 | 795 | # EC2Runner.run() runs the given function, passing through whatever | ||
| 379 | 796 | # arguments and keyword arguments it has been given. | ||
| 380 | 675 | calls = [] | 797 | calls = [] |
| 381 | 676 | runner = self.make_ec2runner() | 798 | runner = self.make_ec2runner() |
| 385 | 677 | runner.run( | 799 | def function(*args, **kwargs): |
| 386 | 678 | "boring test method", | 800 | calls.append((args, kwargs)) |
| 387 | 679 | lambda *a, **kw: calls.append((a, kw)), "foo", "bar", baz="qux") | 801 | runner.run("boring test method", function, "foo", "bar", baz="qux") |
| 388 | 680 | self.assertEqual([(("foo", "bar"), {'baz': 'qux'})], calls) | 802 | self.assertEqual([(("foo", "bar"), {'baz': 'qux'})], calls) |
| 389 | 681 | 803 | ||
| 390 | 682 | def test_email_on_failure_no_emails(self): | 804 | def test_email_on_failure_no_emails(self): |
| 391 | @@ -688,7 +810,7 @@ | |||
| 392 | 688 | self.assertEqual([], log) | 810 | self.assertEqual([], log) |
| 393 | 689 | 811 | ||
| 394 | 690 | def test_email_on_failure_some_emails(self): | 812 | def test_email_on_failure_some_emails(self): |
| 396 | 691 | # If no emails are specified, then no email is sent on failure. | 813 | # If emails *are* specified, then an email is sent on failure. |
| 397 | 692 | log = [] | 814 | log = [] |
| 398 | 693 | runner = self.make_ec2runner( | 815 | runner = self.make_ec2runner( |
| 399 | 694 | email_log=log, emails=["foo@example.com"]) | 816 | email_log=log, emails=["foo@example.com"]) |
| 400 | @@ -701,6 +823,7 @@ | |||
| 401 | 701 | self.assertIn('ZeroDivisionError', str(message)) | 823 | self.assertIn('ZeroDivisionError', str(message)) |
| 402 | 702 | 824 | ||
| 403 | 703 | def test_email_with_launchpad_tester_failure(self): | 825 | def test_email_with_launchpad_tester_failure(self): |
| 404 | 826 | # LaunchpadTester sends email on catastrophic failure. | ||
| 405 | 704 | email_log = [] | 827 | email_log = [] |
| 406 | 705 | to_emails = ['foo@example.com'] | 828 | to_emails = ['foo@example.com'] |
| 407 | 706 | request = self.make_request(emails=to_emails, emails_sent=email_log) | 829 | request = self.make_request(emails=to_emails, emails_sent=email_log) |
| 408 | @@ -767,18 +890,37 @@ | |||
| 409 | 767 | def get_body_text(self, email): | 890 | def get_body_text(self, email): |
| 410 | 768 | return email.get_payload()[0].get_payload() | 891 | return email.get_payload()[0].get_payload() |
| 411 | 769 | 892 | ||
| 412 | 893 | def make_empty_result(self): | ||
| 413 | 894 | return TestResult() | ||
| 414 | 895 | |||
| 415 | 896 | def make_successful_result(self): | ||
| 416 | 897 | result = self.make_empty_result() | ||
| 417 | 898 | result.startTest(self) | ||
| 418 | 899 | result.stopTest(self) | ||
| 419 | 900 | return result | ||
| 420 | 901 | |||
| 421 | 902 | def make_failing_result(self): | ||
| 422 | 903 | result = self.make_empty_result() | ||
| 423 | 904 | result.startTest(self) | ||
| 424 | 905 | try: | ||
| 425 | 906 | 1/0 | ||
| 426 | 907 | except ZeroDivisionError: | ||
| 427 | 908 | result.addError(self, sys.exc_info()) | ||
| 428 | 909 | result.stopTest(self) | ||
| 429 | 910 | return result | ||
| 430 | 911 | |||
| 431 | 770 | def test_success_no_emails(self): | 912 | def test_success_no_emails(self): |
| 432 | 771 | log = [] | 913 | log = [] |
| 433 | 772 | request = self.make_request(emails=[], emails_sent=log) | 914 | request = self.make_request(emails=[], emails_sent=log) |
| 434 | 773 | logger = self.make_logger(request=request) | 915 | logger = self.make_logger(request=request) |
| 436 | 774 | logger.got_result(True) | 916 | logger.got_result(self.make_successful_result()) |
| 437 | 775 | self.assertEqual([], log) | 917 | self.assertEqual([], log) |
| 438 | 776 | 918 | ||
| 439 | 777 | def test_failure_no_emails(self): | 919 | def test_failure_no_emails(self): |
| 440 | 778 | log = [] | 920 | log = [] |
| 441 | 779 | request = self.make_request(emails=[], emails_sent=log) | 921 | request = self.make_request(emails=[], emails_sent=log) |
| 442 | 780 | logger = self.make_logger(request=request) | 922 | logger = self.make_logger(request=request) |
| 444 | 781 | logger.got_result(False) | 923 | logger.got_result(self.make_failing_result()) |
| 445 | 782 | self.assertEqual([], log) | 924 | self.assertEqual([], log) |
| 446 | 783 | 925 | ||
| 447 | 784 | def test_submits_to_pqm_on_success(self): | 926 | def test_submits_to_pqm_on_success(self): |
| 448 | @@ -787,7 +929,7 @@ | |||
| 449 | 787 | request = self.make_request( | 929 | request = self.make_request( |
| 450 | 788 | emails=[], pqm_message=message, emails_sent=log) | 930 | emails=[], pqm_message=message, emails_sent=log) |
| 451 | 789 | logger = self.make_logger(request=request) | 931 | logger = self.make_logger(request=request) |
| 453 | 790 | logger.got_result(True) | 932 | logger.got_result(self.make_successful_result()) |
| 454 | 791 | self.assertEqual([message], log) | 933 | self.assertEqual([message], log) |
| 455 | 792 | 934 | ||
| 456 | 793 | def test_records_pqm_submission_in_email(self): | 935 | def test_records_pqm_submission_in_email(self): |
| 457 | @@ -796,20 +938,20 @@ | |||
| 458 | 796 | request = self.make_request( | 938 | request = self.make_request( |
| 459 | 797 | emails=['foo@example.com'], pqm_message=message, emails_sent=log) | 939 | emails=['foo@example.com'], pqm_message=message, emails_sent=log) |
| 460 | 798 | logger = self.make_logger(request=request) | 940 | logger = self.make_logger(request=request) |
| 462 | 799 | logger.got_result(True) | 941 | logger.got_result(self.make_successful_result()) |
| 463 | 800 | [pqm_message, user_message] = log | 942 | [pqm_message, user_message] = log |
| 464 | 801 | self.assertEqual(message, pqm_message) | 943 | self.assertEqual(message, pqm_message) |
| 465 | 802 | self.assertIn( | 944 | self.assertIn( |
| 466 | 803 | 'SUBMITTED TO PQM:\n%s' % (message['Subject'],), | 945 | 'SUBMITTED TO PQM:\n%s' % (message['Subject'],), |
| 467 | 804 | self.get_body_text(user_message)) | 946 | self.get_body_text(user_message)) |
| 468 | 805 | 947 | ||
| 470 | 806 | def test_doesnt_submit_to_pqm_no_failure(self): | 948 | def test_doesnt_submit_to_pqm_on_failure(self): |
| 471 | 807 | log = [] | 949 | log = [] |
| 472 | 808 | message = {'Subject': 'foo'} | 950 | message = {'Subject': 'foo'} |
| 473 | 809 | request = self.make_request( | 951 | request = self.make_request( |
| 474 | 810 | emails=[], pqm_message=message, emails_sent=log) | 952 | emails=[], pqm_message=message, emails_sent=log) |
| 475 | 811 | logger = self.make_logger(request=request) | 953 | logger = self.make_logger(request=request) |
| 477 | 812 | logger.got_result(False) | 954 | logger.got_result(self.make_failing_result()) |
| 478 | 813 | self.assertEqual([], log) | 955 | self.assertEqual([], log) |
| 479 | 814 | 956 | ||
| 480 | 815 | def test_records_non_pqm_submission_in_email(self): | 957 | def test_records_non_pqm_submission_in_email(self): |
| 481 | @@ -818,7 +960,7 @@ | |||
| 482 | 818 | request = self.make_request( | 960 | request = self.make_request( |
| 483 | 819 | emails=['foo@example.com'], pqm_message=message, emails_sent=log) | 961 | emails=['foo@example.com'], pqm_message=message, emails_sent=log) |
| 484 | 820 | logger = self.make_logger(request=request) | 962 | logger = self.make_logger(request=request) |
| 486 | 821 | logger.got_result(False) | 963 | logger.got_result(self.make_failing_result()) |
| 487 | 822 | [user_message] = log | 964 | [user_message] = log |
| 488 | 823 | self.assertIn( | 965 | self.assertIn( |
| 489 | 824 | '**NOT** submitted to PQM:\n%s' % (message['Subject'],), | 966 | '**NOT** submitted to PQM:\n%s' % (message['Subject'],), |
| 490 | @@ -829,29 +971,25 @@ | |||
| 491 | 829 | request = self.make_request( | 971 | request = self.make_request( |
| 492 | 830 | emails=['foo@example.com'], emails_sent=log) | 972 | emails=['foo@example.com'], emails_sent=log) |
| 493 | 831 | logger = self.make_logger(request=request) | 973 | logger = self.make_logger(request=request) |
| 495 | 832 | logger.got_result(False) | 974 | logger.got_result(self.make_failing_result()) |
| 496 | 833 | [user_message] = log | 975 | [user_message] = log |
| 497 | 834 | self.assertIn( | 976 | self.assertIn( |
| 498 | 835 | '(See the attached file for the complete log)\n', | 977 | '(See the attached file for the complete log)\n', |
| 499 | 836 | self.get_body_text(user_message)) | 978 | self.get_body_text(user_message)) |
| 500 | 837 | 979 | ||
| 502 | 838 | def test_email_body_is_summary(self): | 980 | def test_email_body_is_format_result(self): |
| 503 | 839 | # The body of the email sent to the user is the summary file. | 981 | # The body of the email sent to the user is the summary file. |
| 504 | 840 | # XXX: JonathanLange 2010-08-17: We actually want to change this | ||
| 505 | 841 | # behaviour so that the summary log stays roughly the same as it is | ||
| 506 | 842 | # now and can vary independently from the contents of the sent | ||
| 507 | 843 | # email. We probably just want the contents of the email to be a list | ||
| 508 | 844 | # of failing tests. | ||
| 509 | 845 | log = [] | 982 | log = [] |
| 510 | 846 | request = self.make_request( | 983 | request = self.make_request( |
| 511 | 847 | emails=['foo@example.com'], emails_sent=log) | 984 | emails=['foo@example.com'], emails_sent=log) |
| 512 | 848 | logger = self.make_logger(request=request) | 985 | logger = self.make_logger(request=request) |
| 515 | 849 | logger.get_summary_stream().write('bar\nbaz\nqux\n') | 986 | result = self.make_failing_result() |
| 516 | 850 | logger.got_result(False) | 987 | logger.got_result(result) |
| 517 | 851 | [user_message] = log | 988 | [user_message] = log |
| 518 | 852 | self.assertEqual( | 989 | self.assertEqual( |
| 521 | 853 | 'bar\nbaz\nqux\n\n(See the attached file for the complete log)\n', | 990 | request.format_result( |
| 522 | 854 | self.get_body_text(user_message)) | 991 | result, logger._start_time, logger._end_time), |
| 523 | 992 | self.get_body_text(user_message).decode('quoted-printable')) | ||
| 524 | 855 | 993 | ||
| 525 | 856 | def test_gzip_of_full_log_attached(self): | 994 | def test_gzip_of_full_log_attached(self): |
| 526 | 857 | # The full log is attached to the email. | 995 | # The full log is attached to the email. |
| 527 | @@ -861,7 +999,7 @@ | |||
| 528 | 861 | logger = self.make_logger(request=request) | 999 | logger = self.make_logger(request=request) |
| 529 | 862 | logger.got_line("output from test process\n") | 1000 | logger.got_line("output from test process\n") |
| 530 | 863 | logger.got_line("more output\n") | 1001 | logger.got_line("more output\n") |
| 532 | 864 | logger.got_result(False) | 1002 | logger.got_result(self.make_successful_result()) |
| 533 | 865 | [user_message] = log | 1003 | [user_message] = log |
| 534 | 866 | [body, attachment] = user_message.get_payload() | 1004 | [body, attachment] = user_message.get_payload() |
| 535 | 867 | self.assertEqual('application/x-gzip', attachment['Content-Type']) | 1005 | self.assertEqual('application/x-gzip', attachment['Content-Type']) |
| 536 | 868 | 1006 | ||
| 537 | === modified file 'lib/lp/services/job/tests/test_runner.py' | |||
| 538 | --- lib/lp/services/job/tests/test_runner.py 2010-09-15 22:09:12 +0000 | |||
| 539 | +++ lib/lp/services/job/tests/test_runner.py 2010-09-22 13:36:00 +0000 | |||
| 540 | @@ -216,7 +216,10 @@ | |||
| 541 | 216 | # aborted, it is None. | 216 | # aborted, it is None. |
| 542 | 217 | self.assertIs(None, job.job.log) | 217 | self.assertIs(None, job.job.log) |
| 543 | 218 | 218 | ||
| 545 | 219 | def test_runAll_mails_oopses(self): | 219 | # XXX michaeln 2010-09-22 bug=644984 |
| 546 | 220 | # This is failing sometimes, but has not yet been reproduced | ||
| 547 | 221 | # locally. | ||
| 548 | 222 | def _disabled_test_runAll_mails_oopses(self): | ||
| 549 | 220 | """Email interested parties about OOPses.""" | 223 | """Email interested parties about OOPses.""" |
| 550 | 221 | job_1, job_2 = self.makeTwoJobs() | 224 | job_1, job_2 = self.makeTwoJobs() |
| 551 | 222 | def raiseError(): | 225 | def raiseError(): |
This is nice.
A few thoughts for now, or later, or never.
Perhaps testrepository would make this a lot smaller? Like tiny?
Perhaps Fixtures would make some of the test machinery a little cleaner. (we can use them in LP).