Launchpad itself

Merge lp:~jml/launchpad/distribution-mirror-prober into lp:launchpad

  1. distribution-mirror-prober
  2. Merge into devel
Proposed by Jonathan Lange
Status: Merged
Merged at revision: 12151
Proposed branch: lp:~jml/launchpad/distribution-mirror-prober
Merge into: lp:launchpad
Diff against target: 780 lines (+259/-221)
5 files modified
cronscripts/distributionmirror-prober.py (+13/-136)
lib/lp/registry/browser/tests/test_breadcrumbs.py (+2/-2)
lib/lp/registry/scripts/distributionmirror_prober.py (+153/-3)
lib/lp/registry/tests/test_distributionmirror_prober.py (+88/-79)
lib/lp/testing/factory.py (+3/-1)
To merge this branch: bzr merge lp:~jml/launchpad/distribution-mirror-prober
Reviewer Review Type Date Requested Status
Jelmer Vernooij (community) code Approve
Review via email: mp+44656@code.launchpad.net

Commit message

[r=jelmer][ui=none][no-qa] Move distributionmirror prober logic into the module

Description of the change

I'd like to make the distribution mirror prober look and behave like normal Twisted code. I want it to return Deferreds from methods and to only do reactor.run/stop things at the very top-most level. I want it to clean up after itself.

This branch doesn't do that though. This branch is a very simple branch that just moves code around in an effort to make the prober easier to change.

I've moved the bulk of the logic out of the cronscript and into the module, so that it can be more easily tested and used by other code.

I've also modernized the tests a little, making them inherit from lp.testing base classes, using factories where I can and generally trimming down setUp methods. I've deleted some of the unnecessary reactor-thwomping tearDowns that I added a while back, leaving only the necessary thwomps.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'cronscripts/distributionmirror-prober.py'
2--- cronscripts/distributionmirror-prober.py 2010-11-08 12:52:43 +0000
3+++ cronscripts/distributionmirror-prober.py 2010-12-24 17:11:03 +0000
4@@ -10,53 +10,20 @@
5 import _pythonpath
6
7 import os
8-from StringIO import StringIO
9-
10-from twisted.internet import reactor
11-
12-from zope.component import getUtility
13
14 from canonical.config import config
15-from canonical.database.sqlbase import ISOLATION_LEVEL_AUTOCOMMIT
16 from lp.services.scripts.base import (
17- LaunchpadCronScript, LaunchpadScriptFailure)
18-from lp.registry.interfaces.distributionmirror import (
19- IDistributionMirrorSet,
20- MirrorContent,
21+ LaunchpadCronScript,
22+ LaunchpadScriptFailure,
23 )
24-from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
25-from canonical.launchpad.webapp import canonical_url
26-from lp.registry.scripts.distributionmirror_prober import (
27- get_expected_cdimage_paths, probe_archive_mirror, probe_cdimage_mirror)
28-
29-
30-class DistroMirrorProber(LaunchpadCronScript):
31+from lp.registry.interfaces.distributionmirror import MirrorContent
32+from lp.registry.scripts.distributionmirror_prober import DistroMirrorProber
33+
34+
35+class DistroMirrorProberScript(LaunchpadCronScript):
36 usage = ('%prog --content-type=(archive|cdimage) [--force] '
37 '[--no-owner-notification] [--max-mirrors=N]')
38
39- def _sanity_check_mirror(self, mirror):
40- """Check that the given mirror is official and has an http_base_url.
41- """
42- assert mirror.isOfficial(), (
43- 'Non-official mirrors should not be probed')
44- if mirror.base_url is None:
45- self.logger.warning(
46- "Mirror '%s' of distribution '%s' doesn't have a base URL; "
47- "we can't probe it." % (
48- mirror.name, mirror.distribution.name))
49- return False
50- return True
51-
52- def _create_probe_record(self, mirror, logfile):
53- """Create a probe record for the given mirror with the given logfile.
54- """
55- logfile.seek(0)
56- filename = '%s-probe-logfile.txt' % mirror.name
57- log_file = getUtility(ILibraryFileAliasSet).create(
58- name=filename, size=len(logfile.getvalue()),
59- file=logfile, contentType='text/plain')
60- mirror.newProbeRecord(log_file)
61-
62 def add_my_options(self):
63 self.parser.add_option('--content-type',
64 dest='content_type', default=None, action='store',
65@@ -76,117 +43,27 @@
66
67 def main(self):
68 if self.options.content_type == 'archive':
69- probe_function = probe_archive_mirror
70 content_type = MirrorContent.ARCHIVE
71 elif self.options.content_type == 'cdimage':
72- probe_function = probe_cdimage_mirror
73 content_type = MirrorContent.RELEASE
74 else:
75 raise LaunchpadScriptFailure(
76 'Wrong value for argument --content-type: %s'
77 % self.options.content_type)
78
79- orig_proxy = os.environ.get('http_proxy')
80 if config.distributionmirrorprober.use_proxy:
81 os.environ['http_proxy'] = config.launchpad.http_proxy
82 self.logger.debug("Using %s as proxy." % os.environ['http_proxy'])
83 else:
84 self.logger.debug("Not using any proxy.")
85
86- self.txn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
87- self.txn.begin()
88-
89- # Using a script argument to control a config variable is not a great
90- # idea, but to me this seems better than passing the no_remote_hosts
91- # value through a lot of method/function calls, until it reaches the
92- # probe() method.
93- if self.options.no_remote_hosts:
94- localhost_only_conf = """
95- [distributionmirrorprober]
96- localhost_only: True
97- """
98- config.push('localhost_only_conf', localhost_only_conf)
99-
100- self.logger.info('Probing %s Mirrors' % content_type.title)
101-
102- mirror_set = getUtility(IDistributionMirrorSet)
103-
104- results = mirror_set.getMirrorsToProbe(
105- content_type, ignore_last_probe=self.options.force,
106- limit=self.options.max_mirrors)
107- mirror_ids = [mirror.id for mirror in results]
108- unchecked_keys = []
109- logfiles = {}
110- probed_mirrors = []
111-
112- for mirror_id in mirror_ids:
113- mirror = mirror_set[mirror_id]
114- if not self._sanity_check_mirror(mirror):
115- continue
116-
117- # XXX: salgado 2006-05-26:
118- # Some people registered mirrors on distros other than Ubuntu back
119- # in the old times, so now we need to do this small hack here.
120- if not mirror.distribution.full_functionality:
121- self.logger.info(
122- "Mirror '%s' of distribution '%s' can't be probed --we "
123- "only probe Ubuntu mirrors."
124- % (mirror.name, mirror.distribution.name))
125- continue
126-
127- probed_mirrors.append(mirror)
128- logfile = StringIO()
129- logfiles[mirror_id] = logfile
130- probe_function(mirror, logfile, unchecked_keys, self.logger)
131-
132- if probed_mirrors:
133- reactor.run()
134- self.logger.info('Probed %d mirrors.' % len(probed_mirrors))
135- else:
136- self.logger.info('No mirrors to probe.')
137-
138- disabled_mirrors = []
139- reenabled_mirrors = []
140- # Now that we finished probing all mirrors, we check if any of these
141- # mirrors appear to have no content mirrored, and, if so, mark them as
142- # disabled and notify their owners.
143- expected_iso_images_count = len(get_expected_cdimage_paths())
144- notify_owner = not self.options.no_owner_notification
145- for mirror in probed_mirrors:
146- log = logfiles[mirror.id]
147- self._create_probe_record(mirror, log)
148- if mirror.shouldDisable(expected_iso_images_count):
149- if mirror.enabled:
150- log.seek(0)
151- mirror.disable(notify_owner, log.getvalue())
152- disabled_mirrors.append(canonical_url(mirror))
153- else:
154- # Ensure the mirror is enabled, so that it shows up on public
155- # mirror listings.
156- if not mirror.enabled:
157- mirror.enabled = True
158- reenabled_mirrors.append(canonical_url(mirror))
159-
160- if disabled_mirrors:
161- self.logger.info(
162- 'Disabling %s mirror(s): %s'
163- % (len(disabled_mirrors), ", ".join(disabled_mirrors)))
164- if reenabled_mirrors:
165- self.logger.info(
166- 'Re-enabling %s mirror(s): %s'
167- % (len(reenabled_mirrors), ", ".join(reenabled_mirrors)))
168- # XXX: salgado 2007-04-03:
169- # This should be done in LaunchpadScript.lock_and_run() when
170- # the isolation used is ISOLATION_LEVEL_AUTOCOMMIT. Also note
171- # that replacing this with a flush_database_updates() doesn't
172- # have the same effect, it seems.
173- self.txn.commit()
174-
175- self.logger.info('Done.')
176+ DistroMirrorProber(self.txn, self.logger).probe(
177+ content_type, self.options.no_remote_hosts, self.options.force,
178+ self.options.max_mirrors, not self.options.no_owner_notification)
179
180
181 if __name__ == '__main__':
182- script = DistroMirrorProber('distributionmirror-prober',
183- dbuser=config.distributionmirrorprober.dbuser)
184+ script = DistroMirrorProberScript(
185+ 'distributionmirror-prober',
186+ dbuser=config.distributionmirrorprober.dbuser)
187 script.lock_and_run()
188-
189
190=== modified file 'lib/lp/registry/browser/tests/test_breadcrumbs.py'
191--- lib/lp/registry/browser/tests/test_breadcrumbs.py 2010-12-15 22:05:43 +0000
192+++ lib/lp/registry/browser/tests/test_breadcrumbs.py 2010-12-24 17:11:03 +0000
193@@ -58,7 +58,7 @@
194 http_url=http_url)
195 crumbs = self.getBreadcrumbsForObject(mirror)
196 last_crumb = crumbs[-1]
197- self.assertEqual("Example.com-archive", last_crumb.text)
198+ self.assertEqual(mirror.displayname, last_crumb.text)
199
200 def test_distributionmirror_withFtpUrl(self):
201 # If no displayname, the breadcrumb text will be the mirror name,
202@@ -70,7 +70,7 @@
203 ftp_url=ftp_url)
204 crumbs = self.getBreadcrumbsForObject(mirror)
205 last_crumb = crumbs[-1]
206- self.assertEqual("Example.com-archive", last_crumb.text)
207+ self.assertEqual(mirror.displayname, last_crumb.text)
208
209
210 class TestMilestoneBreadcrumb(BaseBreadcrumbTestCase):
211
212=== modified file 'lib/lp/registry/scripts/distributionmirror_prober.py'
213--- lib/lp/registry/scripts/distributionmirror_prober.py 2010-12-13 15:44:15 +0000
214+++ lib/lp/registry/scripts/distributionmirror_prober.py 2010-12-24 17:11:03 +0000
215@@ -1,11 +1,17 @@
216-# Copyright 2009 Canonical Ltd. This software is licensed under the
217+# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
218 # GNU Affero General Public License version 3 (see the file LICENSE).
219
220+__metaclass__ = type
221+__all__ = [
222+ 'DistroMirrorProber',
223+ ]
224+
225 from datetime import datetime
226 import httplib
227 import itertools
228 import logging
229 import os
230+from StringIO import StringIO
231 import urllib2
232 import urlparse
233
234@@ -20,8 +26,13 @@
235 from zope.component import getUtility
236
237 from canonical.config import config
238+from canonical.database.sqlbase import ISOLATION_LEVEL_AUTOCOMMIT
239 from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities
240+from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
241+from canonical.launchpad.webapp import canonical_url
242 from lp.registry.interfaces.distributionmirror import (
243+ IDistributionMirrorSet,
244+ MirrorContent,
245 MirrorFreshness,
246 UnableToFetchCDImageFileList,
247 )
248@@ -55,8 +66,6 @@
249 # start connecting.
250 OVERALL_REQUESTS = 100
251
252-__metaclass__ = type
253-
254
255 class LoggingMixin:
256 """Common logging class for archive and releases mirror messages."""
257@@ -752,3 +761,144 @@
258 assert port.isdigit()
259 port = int(port)
260 return scheme, host, port, path
261+
262+
263+class DistroMirrorProber:
264+ """Main entry point for the distribution mirror prober."""
265+
266+ def __init__(self, txn, logger):
267+ self.txn = txn
268+ self.logger = logger
269+
270+ def _sanity_check_mirror(self, mirror):
271+ """Check that the given mirror is official and has an http_base_url.
272+ """
273+ assert mirror.isOfficial(), (
274+ 'Non-official mirrors should not be probed')
275+ if mirror.base_url is None:
276+ self.logger.warning(
277+ "Mirror '%s' of distribution '%s' doesn't have a base URL; "
278+ "we can't probe it." % (
279+ mirror.name, mirror.distribution.name))
280+ return False
281+ return True
282+
283+ def _create_probe_record(self, mirror, logfile):
284+ """Create a probe record for the given mirror with the given logfile.
285+ """
286+ logfile.seek(0)
287+ filename = '%s-probe-logfile.txt' % mirror.name
288+ log_file = getUtility(ILibraryFileAliasSet).create(
289+ name=filename, size=len(logfile.getvalue()),
290+ file=logfile, contentType='text/plain')
291+ mirror.newProbeRecord(log_file)
292+
293+ def probe(self, content_type, no_remote_hosts, ignore_last_probe,
294+ max_mirrors, notify_owner):
295+ """Probe distribution mirrors.
296+
297+ :param content_type: The type of mirrored content, as a
298+ `MirrorContent`.
299+ :param no_remote_hosts: If True, restrict access to localhost.
300+ :param ignore_last_probe: If True, ignore the results of the last
301+ probe and probe again anyway.
302+ :param max_mirrors: The maximum number of mirrors to probe. If None,
303+ no maximum.
304+ :param notify_owner: Send failure notification to the owners of the
305+ mirrors.
306+ """
307+ if content_type == MirrorContent.ARCHIVE:
308+ probe_function = probe_archive_mirror
309+ elif content_type == MirrorContent.RELEASE:
310+ probe_function = probe_cdimage_mirror
311+ else:
312+ raise ValueError(
313+ "Unrecognized content_type: %s" % (content_type,))
314+
315+ self.txn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
316+ self.txn.begin()
317+
318+ # To me this seems better than passing the no_remote_hosts value
319+ # through a lot of method/function calls, until it reaches the probe()
320+ # method. (salgado)
321+ if no_remote_hosts:
322+ localhost_only_conf = """
323+ [distributionmirrorprober]
324+ localhost_only: True
325+ """
326+ config.push('localhost_only_conf', localhost_only_conf)
327+
328+ self.logger.info('Probing %s Mirrors' % content_type.title)
329+
330+ mirror_set = getUtility(IDistributionMirrorSet)
331+ results = mirror_set.getMirrorsToProbe(
332+ content_type, ignore_last_probe=ignore_last_probe,
333+ limit=max_mirrors)
334+ mirror_ids = [mirror.id for mirror in results]
335+ unchecked_keys = []
336+ logfiles = {}
337+ probed_mirrors = []
338+
339+ for mirror_id in mirror_ids:
340+ mirror = mirror_set[mirror_id]
341+ if not self._sanity_check_mirror(mirror):
342+ continue
343+
344+ # XXX: salgado 2006-05-26:
345+ # Some people registered mirrors on distros other than Ubuntu back
346+ # in the old times, so now we need to do this small hack here.
347+ if not mirror.distribution.full_functionality:
348+ self.logger.info(
349+ "Mirror '%s' of distribution '%s' can't be probed --we "
350+ "only probe Ubuntu mirrors."
351+ % (mirror.name, mirror.distribution.name))
352+ continue
353+
354+ probed_mirrors.append(mirror)
355+ logfile = StringIO()
356+ logfiles[mirror_id] = logfile
357+ probe_function(mirror, logfile, unchecked_keys, self.logger)
358+
359+ if probed_mirrors:
360+ reactor.run()
361+ self.logger.info('Probed %d mirrors.' % len(probed_mirrors))
362+ else:
363+ self.logger.info('No mirrors to probe.')
364+
365+ disabled_mirrors = []
366+ reenabled_mirrors = []
367+ # Now that we finished probing all mirrors, we check if any of these
368+ # mirrors appear to have no content mirrored, and, if so, mark them as
369+ # disabled and notify their owners.
370+ expected_iso_images_count = len(get_expected_cdimage_paths())
371+ for mirror in probed_mirrors:
372+ log = logfiles[mirror.id]
373+ self._create_probe_record(mirror, log)
374+ if mirror.shouldDisable(expected_iso_images_count):
375+ if mirror.enabled:
376+ log.seek(0)
377+ mirror.disable(notify_owner, log.getvalue())
378+ disabled_mirrors.append(canonical_url(mirror))
379+ else:
380+ # Ensure the mirror is enabled, so that it shows up on public
381+ # mirror listings.
382+ if not mirror.enabled:
383+ mirror.enabled = True
384+ reenabled_mirrors.append(canonical_url(mirror))
385+
386+ if disabled_mirrors:
387+ self.logger.info(
388+ 'Disabling %s mirror(s): %s'
389+ % (len(disabled_mirrors), ", ".join(disabled_mirrors)))
390+ if reenabled_mirrors:
391+ self.logger.info(
392+ 'Re-enabling %s mirror(s): %s'
393+ % (len(reenabled_mirrors), ", ".join(reenabled_mirrors)))
394+ # XXX: salgado 2007-04-03:
395+ # This should be done in LaunchpadScript.lock_and_run() when
396+ # the isolation used is ISOLATION_LEVEL_AUTOCOMMIT. Also note
397+ # that replacing this with a flush_database_updates() doesn't
398+ # have the same effect, it seems.
399+ self.txn.commit()
400+
401+ self.logger.info('Done.')
402
403=== modified file 'lib/lp/registry/tests/test_distributionmirror_prober.py'
404--- lib/lp/registry/tests/test_distributionmirror_prober.py 2010-12-10 21:10:42 +0000
405+++ lib/lp/registry/tests/test_distributionmirror_prober.py 2010-12-24 17:11:03 +0000
406@@ -1,8 +1,6 @@
407-# Copyright 2009 Canonical Ltd. This software is licensed under the
408+# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
409 # GNU Affero General Public License version 3 (see the file LICENSE).
410
411-# pylint: disable-msg=W0703
412-
413 """distributionmirror-prober tests."""
414
415 __metaclass__ = type
416@@ -13,7 +11,6 @@
417 import logging
418 import os
419 from StringIO import StringIO
420-import unittest
421
422 from lazr.uri import URI
423 from sqlobject import SQLObjectNotFound
424@@ -25,16 +22,19 @@
425 from twisted.trial.unittest import TestCase as TrialTestCase
426 from twisted.web import server
427
428+from zope.component import getUtility
429+from zope.security.proxy import removeSecurityProxy
430+
431 import canonical
432 from canonical.config import config
433 from canonical.launchpad.daemons.tachandler import TacTestSetup
434+from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities
435 from canonical.testing.layers import (
436- LaunchpadZopelessLayer,
437 TwistedLayer,
438+ ZopelessDatabaseLayer,
439 )
440 from lp.registry.interfaces.pocket import PackagePublishingPocket
441 from lp.registry.model.distributionmirror import DistributionMirror
442-from lp.registry.model.distroseries import DistroSeries
443 from lp.registry.scripts import distributionmirror_prober
444 from lp.registry.scripts.distributionmirror_prober import (
445 _build_request_for_cdimage_file_list,
446@@ -64,6 +64,10 @@
447 from lp.registry.tests.distributionmirror_http_server import (
448 DistributionMirrorTestHTTPServer,
449 )
450+from lp.testing import (
451+ TestCase,
452+ TestCaseWithFactory,
453+ )
454
455
456 class HTTPServerTestSetup(TacTestSetup):
457@@ -276,7 +280,7 @@
458 self.redirectedTo = url
459
460
461-class TestProberFactoryRequestTimeoutRatioWithoutTwisted(unittest.TestCase):
462+class TestProberFactoryRequestTimeoutRatioWithoutTwisted(TestCase):
463 """Tests to ensure we stop issuing requests on a given host if the
464 requests/timeouts ratio on that host is too low.
465
466@@ -288,6 +292,7 @@
467 host = 'foo.bar'
468
469 def setUp(self):
470+ super(TestProberFactoryRequestTimeoutRatioWithoutTwisted, self).setUp()
471 self.orig_host_requests = dict(
472 distributionmirror_prober.host_requests)
473 self.orig_host_timeouts = dict(
474@@ -297,6 +302,7 @@
475 # Restore the globals that our tests fiddle with.
476 distributionmirror_prober.host_requests = self.orig_host_requests
477 distributionmirror_prober.host_timeouts = self.orig_host_timeouts
478+ super(TestProberFactoryRequestTimeoutRatioWithoutTwisted, self).tearDown()
479
480 def _createProberStubConnectAndProbe(self, requests, timeouts):
481 """Create a ProberFactory object with a URL inside self.host and call
482@@ -431,9 +437,10 @@
483 return self.assertFailure(d, ConnectionSkipped)
484
485
486-class TestMultiLock(unittest.TestCase):
487+class TestMultiLock(TestCase):
488
489 def setUp(self):
490+ super(TestMultiLock, self).setUp()
491 self.lock_one = defer.DeferredLock()
492 self.lock_two = defer.DeferredLock()
493 self.multi_lock = MultiLock(self.lock_one, self.lock_two)
494@@ -510,7 +517,7 @@
495 self.assertEquals(self.count, 1, "self.callback should have run.")
496
497
498-class TestRedirectAwareProberFactoryAndProtocol(unittest.TestCase):
499+class TestRedirectAwareProberFactoryAndProtocol(TestCase):
500
501 def test_redirect_resets_timeout(self):
502 prober = RedirectAwareProberFactory('http://foo.bar')
503@@ -599,43 +606,46 @@
504 self.failUnless(protocol.transport.disconnecting)
505
506
507-class TestMirrorCDImageProberCallbacks(unittest.TestCase):
508- layer = LaunchpadZopelessLayer
509-
510- def setUp(self):
511- self.layer.switchDbUser(config.distributionmirrorprober.dbuser)
512- self.logger = logging.getLogger('distributionmirror-prober')
513- self.logger.errorCalled = False
514+class TestMirrorCDImageProberCallbacks(TestCaseWithFactory):
515+
516+ layer = ZopelessDatabaseLayer
517+
518+ def makeMirrorProberCallbacks(self):
519+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
520+ distroseries = removeSecurityProxy(
521+ self.factory.makeDistroSeries(distribution=ubuntu))
522+ mirror = removeSecurityProxy(
523+ self.factory.makeMirror(distroseries.distribution))
524+ callbacks = MirrorCDImageProberCallbacks(
525+ mirror, distroseries, 'ubuntu', StringIO())
526+ return callbacks
527+
528+ def getLogger(self):
529+ logger = logging.getLogger('distributionmirror-prober')
530+ logger.errorCalled = False
531 def error(msg):
532- self.logger.errorCalled = True
533- self.logger.error = error
534- mirror = DistributionMirror.get(1)
535- warty = DistroSeries.get(1)
536- flavour = 'ubuntu'
537- log_file = StringIO()
538- self.callbacks = MirrorCDImageProberCallbacks(
539- mirror, warty, flavour, log_file)
540-
541- def tearDown(self):
542- # XXX: JonathanLange 2010-11-22: These tests leave stacks of delayed
543- # calls around. They need to be updated to use Twisted correctly.
544- # For the meantime, just blat the reactor.
545- for delayed_call in reactor.getDelayedCalls():
546- delayed_call.cancel()
547-
548- def test_mirrorcdimageseries_creation_and_deletion(self):
549- callbacks = self.callbacks
550+ logger.errorCalled = True
551+ logger.error = error
552+ return logger
553+
554+ def test_mirrorcdimageseries_creation_and_deletion_all_success(self):
555+ callbacks = self.makeMirrorProberCallbacks()
556 all_success = [(defer.SUCCESS, '200'), (defer.SUCCESS, '200')]
557 mirror_cdimage_series = callbacks.ensureOrDeleteMirrorCDImageSeries(
558 all_success)
559- self.failUnless(
560- mirror_cdimage_series is not None,
561+ self.assertIsNot(
562+ mirror_cdimage_series, None,
563 "If the prober gets a list of 200 Okay statuses, a new "
564 "MirrorCDImageSeries should be created.")
565
566+ def test_mirrorcdimageseries_creation_and_deletion_some_404s(self):
567 not_all_success = [
568 (defer.FAILURE, Failure(BadResponseCode(str(httplib.NOT_FOUND)))),
569 (defer.SUCCESS, '200')]
570+ callbacks = self.makeMirrorProberCallbacks()
571+ all_success = [(defer.SUCCESS, '200'), (defer.SUCCESS, '200')]
572+ mirror_cdimage_series = callbacks.ensureOrDeleteMirrorCDImageSeries(
573+ all_success)
574 callbacks.ensureOrDeleteMirrorCDImageSeries(not_all_success)
575 # If the prober gets at least one 404 status, we need to make sure
576 # there's no MirrorCDImageSeries for that series and flavour.
577@@ -648,8 +658,10 @@
578 # ignored by ensureOrDeleteMirrorCDImageSeries() because they've been
579 # logged by logMissingURL() already and they're expected to happen
580 # some times.
581- self.failUnlessEqual(
582- set(self.callbacks.expected_failures),
583+ logger = self.getLogger()
584+ callbacks = self.makeMirrorProberCallbacks()
585+ self.assertEqual(
586+ set(callbacks.expected_failures),
587 set([
588 BadResponseCode,
589 ProberTimeout,
590@@ -661,65 +673,63 @@
591 ConnectionSkipped(),
592 UnknownURLSchemeAfterRedirect('https://localhost')]
593 for exception in exceptions:
594- failure = self.callbacks.ensureOrDeleteMirrorCDImageSeries(
595+ failure = callbacks.ensureOrDeleteMirrorCDImageSeries(
596 [(defer.FAILURE, Failure(exception))])
597 # Twisted callbacks may raise or return a failure; that's why we
598 # check the return value.
599 self.failIf(isinstance(failure, Failure))
600 # Also, these failures are not logged to stdout/stderr since
601 # they're expected to happen.
602- self.failIf(self.logger.errorCalled)
603+ self.failIf(logger.errorCalled)
604
605 def test_unexpected_failures_are_logged_but_not_raised(self):
606 # Errors which are not expected as logged using the
607 # prober's logger to make sure people see it while still alowing other
608 # mirrors to be probed.
609- failure = self.callbacks.ensureOrDeleteMirrorCDImageSeries(
610+ logger = self.getLogger()
611+ callbacks = self.makeMirrorProberCallbacks()
612+ failure = callbacks.ensureOrDeleteMirrorCDImageSeries(
613 [(defer.FAILURE, Failure(ZeroDivisionError()))])
614 # Twisted callbacks may raise or return a failure; that's why we
615 # check the return value.
616 self.failIf(isinstance(failure, Failure))
617 # Unlike the expected failures, these ones must be logged as errors to
618 # stdout/stderr.
619- self.failUnless(self.logger.errorCalled)
620-
621-
622-class TestArchiveMirrorProberCallbacks(unittest.TestCase):
623- layer = LaunchpadZopelessLayer
624-
625- def setUp(self):
626- mirror = DistributionMirror.get(1)
627- warty = DistroSeries.get(1)
628- pocket = PackagePublishingPocket.RELEASE
629- component = warty.components[0]
630- log_file = StringIO()
631- url = 'foo'
632- self.callbacks = ArchiveMirrorProberCallbacks(
633- mirror, warty, pocket, component, url, log_file)
634-
635- def tearDown(self):
636- # XXX: JonathanLange 2010-11-22: These tests leave stacks of delayed
637- # calls around. They need to be updated to use Twisted correctly.
638- # For the meantime, just blat the reactor.
639- for delayed_call in reactor.getDelayedCalls():
640- delayed_call.cancel()
641+ self.failUnless(logger.errorCalled)
642+
643+
644+class TestArchiveMirrorProberCallbacks(TestCaseWithFactory):
645+ layer = ZopelessDatabaseLayer
646+
647+ def makeMirrorProberCallbacks(self):
648+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
649+ distroseries = removeSecurityProxy(
650+ self.factory.makeDistroSeries(distribution=ubuntu))
651+ mirror = removeSecurityProxy(
652+ self.factory.makeMirror(distroseries.distribution))
653+ component = self.factory.makeComponent()
654+ callbacks = ArchiveMirrorProberCallbacks(
655+ mirror, distroseries, PackagePublishingPocket.RELEASE,
656+ component, 'foo', StringIO())
657+ return callbacks
658
659 def test_failure_propagation(self):
660 # Make sure that deleteMirrorSeries() does not propagate
661 # ProberTimeout, BadResponseCode or ConnectionSkipped failures.
662+ callbacks = self.makeMirrorProberCallbacks()
663 try:
664- self.callbacks.deleteMirrorSeries(
665+ callbacks.deleteMirrorSeries(
666 Failure(ProberTimeout('http://localhost/', 5)))
667 except Exception, e:
668 self.fail("A timeout shouldn't be propagated. Got %s" % e)
669 try:
670- self.callbacks.deleteMirrorSeries(
671+ callbacks.deleteMirrorSeries(
672 Failure(BadResponseCode(str(httplib.INTERNAL_SERVER_ERROR))))
673 except Exception, e:
674 self.fail(
675 "A bad response code shouldn't be propagated. Got %s" % e)
676 try:
677- self.callbacks.deleteMirrorSeries(Failure(ConnectionSkipped()))
678+ callbacks.deleteMirrorSeries(Failure(ConnectionSkipped()))
679 except Exception, e:
680 self.fail("A ConnectionSkipped exception shouldn't be "
681 "propagated. Got %s" % e)
682@@ -727,7 +737,7 @@
683 # Make sure that deleteMirrorSeries() propagate any failure that is
684 # not a ProberTimeout, a BadResponseCode or a ConnectionSkipped.
685 d = defer.Deferred()
686- d.addErrback(self.callbacks.deleteMirrorSeries)
687+ d.addErrback(callbacks.deleteMirrorSeries)
688 def got_result(result):
689 self.fail(
690 "Any failure that's not a timeout/bad-response/skipped "
691@@ -740,15 +750,16 @@
692 self.assertEqual([1], ok)
693
694 def test_mirrorseries_creation_and_deletion(self):
695- mirror_distro_series_source = self.callbacks.ensureMirrorSeries(
696+ callbacks = self.makeMirrorProberCallbacks()
697+ mirror_distro_series_source = callbacks.ensureMirrorSeries(
698 str(httplib.OK))
699- self.failUnless(
700- mirror_distro_series_source is not None,
701+ self.assertIsNot(
702+ mirror_distro_series_source, None,
703 "If the prober gets a 200 Okay status, a new "
704 "MirrorDistroSeriesSource/MirrorDistroArchSeries should be "
705 "created.")
706
707- self.callbacks.deleteMirrorSeries(
708+ callbacks.deleteMirrorSeries(
709 Failure(BadResponseCode(str(httplib.NOT_FOUND))))
710 # If the prober gets a 404 status, we need to make sure there's no
711 # MirrorDistroSeriesSource/MirrorDistroArchSeries referent to
712@@ -758,13 +769,14 @@
713 mirror_distro_series_source.id)
714
715
716-class TestProbeFunctionSemaphores(unittest.TestCase):
717+class TestProbeFunctionSemaphores(TestCase):
718 """Make sure we use one DeferredSemaphore for each hostname when probing
719 mirrors.
720 """
721- layer = LaunchpadZopelessLayer
722+ layer = ZopelessDatabaseLayer
723
724 def setUp(self):
725+ super(TestProbeFunctionSemaphores, self).setUp()
726 self.logger = None
727 # RequestManager uses a mutable class attribute (host_locks) to ensure
728 # all of its instances share the same locks. We don't want our tests
729@@ -778,6 +790,7 @@
730 # For the meantime, just blat the reactor.
731 for delayed_call in reactor.getDelayedCalls():
732 delayed_call.cancel()
733+ super(TestProbeFunctionSemaphores, self).tearDown()
734
735 def test_MirrorCDImageSeries_records_are_deleted_before_probing(self):
736 mirror = DistributionMirror.byName('releases-mirror2')
737@@ -854,7 +867,7 @@
738 restore_http_proxy(orig_proxy)
739
740
741-class TestCDImageFileListFetching(unittest.TestCase):
742+class TestCDImageFileListFetching(TestCase):
743
744 def test_no_cache(self):
745 url = 'http://releases.ubuntu.com/.manifest'
746@@ -863,7 +876,7 @@
747 self.failUnlessEqual(request.headers['Cache-control'], 'no-cache')
748
749
750-class TestLoggingMixin(unittest.TestCase):
751+class TestLoggingMixin(TestCase):
752
753 def _fake_gettime(self):
754 # Fake the current time.
755@@ -888,7 +901,3 @@
756 logger.log_file.seek(0)
757 message = logger.log_file.read()
758 self.assertNotEqual(None, message)
759-
760-
761-def test_suite():
762- return unittest.TestLoader().loadTestsFromName(__name__)
763
764=== modified file 'lib/lp/testing/factory.py'
765--- lib/lp/testing/factory.py 2010-12-20 07:51:34 +0000
766+++ lib/lp/testing/factory.py 2010-12-24 17:11:03 +0000
767@@ -2651,10 +2651,12 @@
768 proberecord = mirror.newProbeRecord(library_alias)
769 return proberecord
770
771- def makeMirror(self, distribution, displayname, country=None,
772+ def makeMirror(self, distribution, displayname=None, country=None,
773 http_url=None, ftp_url=None, rsync_url=None,
774 official_candidate=False):
775 """Create a mirror for the distribution."""
776+ if displayname is None:
777+ displayname = self.getUniqueString("mirror")
778 # If no URL is specified create an HTTP URL.
779 if http_url is None and ftp_url is None and rsync_url is None:
780 http_url = self.getUniqueURL()