Launchpad itself

Merge lp:~jml/launchpad/buildd-deferred-fo-sho into lp:launchpad

  1. buildd-deferred-fo-sho
  2. Merge into devel
Proposed by Jonathan Lange
Status: Merged
Merged at revision: 11801
Proposed branch: lp:~jml/launchpad/buildd-deferred-fo-sho
Merge into: lp:launchpad
Prerequisite: lp:~jml/launchpad/buildd-slavescanner-bustage
Diff against target: 770 lines (+307/-160)
7 files modified
lib/lp/buildmaster/manager.py (+25/-19)
lib/lp/buildmaster/model/builder.py (+38/-17)
lib/lp/buildmaster/tests/test_builder.py (+138/-43)
lib/lp/code/model/recipebuilder.py (+30/-27)
lib/lp/soyuz/model/binarypackagebuildbehavior.py (+56/-40)
lib/lp/soyuz/tests/soyuzbuilddhelpers.py (+8/-4)
lib/lp/translations/model/translationtemplatesbuildbehavior.py (+12/-10)
To merge this branch: bzr merge lp:~jml/launchpad/buildd-deferred-fo-sho
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+36188@code.launchpad.net

Description of the change

Use Deferreds.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/manager.py'
2--- lib/lp/buildmaster/manager.py 2010-09-20 10:21:32 +0000
3+++ lib/lp/buildmaster/manager.py 2010-09-22 11:33:47 +0000
4@@ -283,15 +283,15 @@
5 """Scan the builder and dispatch to it or deal with failures."""
6 self.logger.debug("Scanning builder: %s" % self.builder_name)
7
8- try:
9- slave = self.scan()
10+ d = self.scan()
11+
12+ def got_slave(slave):
13 if slave is None:
14- self.scheduleNextScanCycle()
15+ return self.scheduleNextScanCycle()
16 else:
17- # XXX: Ought to return Deferred.
18- self.resumeAndDispatch(slave)
19- except:
20- error = Failure()
21+ return self.resumeAndDispatch(slave)
22+
23+ def disaster(error):
24 self.logger.info("Scanning failed with: %s\n%s" %
25 (error.getErrorMessage(), error.getTraceback()))
26
27@@ -307,7 +307,11 @@
28 assessFailureCounts(builder, error.getErrorMessage())
29 transaction.commit()
30
31- self.scheduleNextScanCycle()
32+ return self.scheduleNextScanCycle()
33+
34+ d.addCallback(got_slave)
35+ d.addErrback(disaster)
36+ return d
37
38 @write_transaction
39 def scan(self):
40@@ -346,7 +350,7 @@
41 if self.builder.manual:
42 self.logger.debug(
43 '%s is in manual mode, not dispatching.' % self.builder.name)
44- return None
45+ return defer.succeed(None)
46
47 # If the builder is marked unavailable, don't dispatch anything.
48 # Additionaly, because builders can be removed from the pool at
49@@ -362,7 +366,7 @@
50 "job" % self.builder.name)
51 job.reset()
52 transaction.commit()
53- return None
54+ return defer.succeed(None)
55
56 # See if there is a job we can dispatch to the builder slave.
57
58@@ -374,15 +378,17 @@
59 self.builder.name, self.builder.url, self.builder.vm_host)
60 # XXX: Passing buildd_slave=slave overwrites the 'slave' property of
61 # self.builder. Not sure why this is needed yet.
62- self.builder.findAndStartJob(buildd_slave=slave)
63- if self.builder.currentjob is not None:
64- # After a successful dispatch we can reset the
65- # failure_count.
66- self.builder.resetFailureCount()
67- transaction.commit()
68- return slave
69-
70- return None
71+ d = self.builder.findAndStartJob(buildd_slave=slave)
72+ def job_started(candidate):
73+ if self.builder.currentjob is not None:
74+ # After a successful dispatch we can reset the
75+ # failure_count.
76+ self.builder.resetFailureCount()
77+ transaction.commit()
78+ return slave
79+ else:
80+ return None
81+ return d.addCallback(job_started)
82
83 def resumeAndDispatch(self, slave):
84 """Chain the resume and dispatching Deferreds."""
85
86=== modified file 'lib/lp/buildmaster/model/builder.py'
87--- lib/lp/buildmaster/model/builder.py 2010-09-22 11:33:46 +0000
88+++ lib/lp/buildmaster/model/builder.py 2010-09-22 11:33:47 +0000
89@@ -34,6 +34,7 @@
90 Count,
91 Sum,
92 )
93+from twisted.internet import defer
94 from zope.component import getUtility
95 from zope.interface import implements
96
97@@ -166,8 +167,10 @@
98 return self._server.status()
99
100 def ensurepresent(self, sha1sum, url, username, password):
101+ # XXX: Nothing external calls this. Make it private.
102 """Attempt to ensure the given file is present."""
103- return self._server.ensurepresent(sha1sum, url, username, password)
104+ return defer.succeed(
105+ self._server.ensurepresent(sha1sum, url, username, password))
106
107 def getFile(self, sha_sum):
108 """Construct a file-like object to return the named file."""
109@@ -206,13 +209,15 @@
110 logger.debug("Asking builder on %s to ensure it has file %s "
111 "(%s, %s)" % (self.urlbase, libraryfilealias.filename,
112 url, libraryfilealias.content.sha1))
113- self.sendFileToSlave(libraryfilealias.content.sha1, url)
114+ return self.sendFileToSlave(libraryfilealias.content.sha1, url)
115
116 def sendFileToSlave(self, sha1, url, username="", password=""):
117 """Helper to send the file at 'url' with 'sha1' to this builder."""
118- present, info = self.ensurepresent(sha1, url, username, password)
119- if not present:
120- raise CannotFetchFile(url, info)
121+ d = self.ensurepresent(sha1, url, username, password)
122+ def check_present((present, info)):
123+ if not present:
124+ raise CannotFetchFile(url, info)
125+ return d.addCallback(check_present)
126
127 def build(self, buildid, builder_type, chroot_sha1, filemap, args):
128 """Build a thing on this build slave.
129@@ -469,14 +474,19 @@
130
131 # Do it.
132 build_queue_item.markAsBuilding(self)
133- try:
134- self.current_build_behavior.dispatchBuildToSlave(
135- build_queue_item.id, logger)
136- except BuildSlaveFailure, e:
137- logger.debug("Disabling builder: %s" % self.url, exc_info=1)
138+
139+ d = self.current_build_behavior.dispatchBuildToSlave(
140+ build_queue_item.id, logger)
141+
142+ def eb_slave_failure(failure):
143+ failure.trap(BuildSlaveFailure)
144+ e = failure.value
145 self.failBuilder(
146 "Exception (%s) when setting up to new job" % (e,))
147- except CannotFetchFile, e:
148+
149+ def eb_cannot_fetch_file(failure):
150+ failure.trap(CannotFetchFile)
151+ e = failure.value
152 message = """Slave '%s' (%s) was unable to fetch file.
153 ****** URL ********
154 %s
155@@ -485,11 +495,19 @@
156 *******************
157 """ % (self.name, self.url, e.file_url, e.error_information)
158 raise BuildDaemonError(message)
159- except socket.error, e:
160+
161+ def eb_socket_error(failure):
162+ failure.trap(socket.error)
163+ e = failure.value
164 error_message = "Exception (%s) when setting up new job" % (e,)
165 self.handleTimeout(logger, error_message)
166 raise BuildSlaveFailure
167
168+ d.addErrback(eb_slave_failure)
169+ d.addErrback(eb_cannot_fetch_file)
170+ d.addErrback(eb_socket_error)
171+ return d
172+
173 def failBuilder(self, reason):
174 """See IBuilder"""
175 # XXX cprov 2007-04-17: ideally we should be able to notify the
176@@ -682,10 +700,13 @@
177 :param candidate: The job to dispatch.
178 """
179 logger = self._getSlaveScannerLogger()
180- try:
181- self.startBuild(candidate, logger)
182- except (BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch), err:
183+ d = self.startBuild(candidate, logger)
184+ def warn_on_error(failure):
185+ failure.trap(
186+ BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch)
187+ err = failure.value
188 logger.warn('Could not build: %s' % err)
189+ return d.addErrback(warn_on_error)
190
191 def handleTimeout(self, logger, error_message):
192 """See IBuilder."""
193@@ -726,8 +747,8 @@
194 if buildd_slave is not None:
195 self.setSlaveForTesting(buildd_slave)
196
197- self._dispatchBuildCandidate(candidate)
198- return candidate
199+ d = self._dispatchBuildCandidate(candidate)
200+ return d.addCallback(lambda ignored: candidate)
201
202 def getBuildQueue(self):
203 """See `IBuilder`."""
204
205=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
206--- lib/lp/buildmaster/tests/test_builder.py 2010-09-21 16:23:35 +0000
207+++ lib/lp/buildmaster/tests/test_builder.py 2010-09-22 11:33:47 +0000
208@@ -8,8 +8,10 @@
209 import socket
210 import xmlrpclib
211
212-from testtools.content import Content
213-from testtools.content_type import UTF8_TEXT
214+import fixtures
215+
216+from twisted.trial.unittest import TestCase as TrialTestCase
217+from twisted.web.client import getPage
218
219 from zope.component import getUtility
220 from zope.security.proxy import removeSecurityProxy
221@@ -24,10 +26,16 @@
222 )
223 from canonical.testing.layers import (
224 DatabaseFunctionalLayer,
225- LaunchpadZopelessLayer
226+ LaunchpadZopelessLayer,
227+ TwistedLaunchpadZopelessLayer,
228+ TwistedLayer,
229 )
230 from lp.buildmaster.enums import BuildStatus
231-from lp.buildmaster.interfaces.builder import IBuilder, IBuilderSet
232+from lp.buildmaster.interfaces.builder import (
233+ CannotFetchFile,
234+ IBuilder,
235+ IBuilderSet,
236+ )
237 from lp.buildmaster.interfaces.buildfarmjobbehavior import (
238 IBuildFarmJobBehavior,
239 )
240@@ -49,9 +57,12 @@
241 )
242 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
243 from lp.testing import (
244- TestCase,
245+ ANONYMOUS,
246+ login_as,
247+ logout,
248 TestCaseWithFactory,
249 )
250+from lp.testing.factory import LaunchpadObjectFactory
251 from lp.testing.fakemethod import FakeMethod
252
253
254@@ -467,19 +478,11 @@
255 self.builder.current_build_behavior, BinaryPackageBuildBehavior)
256
257
258-class TestSlave(TestCase):
259- """
260- Integration tests for BuilderSlave that verify how it works against a
261- real slave server.
262- """
263-
264- # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
265- # BuilderSlave in buildd-slave.txt and in other places. The tests here
266- # ought to become the canonical tests for BuilderSlave vs running buildd
267- # XML-RPC server interaction.
268+class SlaveTestHelpers(fixtures.Fixture):
269
270 # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`.
271- TEST_URL = 'http://localhost:8221/rpc/'
272+ BASE_URL = 'http://localhost:8221'
273+ TEST_URL = '%s/rpc/' % (BASE_URL,)
274
275 def getServerSlave(self):
276 """Set up a test build slave server.
277@@ -488,11 +491,14 @@
278 """
279 tachandler = BuilddSlaveTestSetup()
280 tachandler.setUp()
281- def addLogFile(exc_info):
282- self.addDetail(
283- 'xmlrpc-log-file',
284- Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))
285- self.addOnException(addLogFile)
286+ # Basically impossible to do this w/ TrialTestCase. But it would be
287+ # really nice to keep it.
288+ #
289+ # def addLogFile(exc_info):
290+ # self.addDetail(
291+ # 'xmlrpc-log-file',
292+ # Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))
293+ # self.addOnException(addLogFile)
294 self.addCleanup(tachandler.tearDown)
295 return tachandler
296
297@@ -527,7 +533,7 @@
298 :return: The build id returned by the slave.
299 """
300 if build_id is None:
301- build_id = self.getUniqueString()
302+ build_id = 'random-build-id'
303 tachandler = self.getServerSlave()
304 chroot_file = 'fake-chroot'
305 dsc_file = 'thing'
306@@ -537,10 +543,30 @@
307 build_id, 'debian', chroot_file, {'.dsc': dsc_file},
308 {'ogrecomponent': 'main'})
309
310+
311+class TestSlave(TrialTestCase):
312+ """
313+ Integration tests for BuilderSlave that verify how it works against a
314+ real slave server.
315+ """
316+
317+ layer = TwistedLayer
318+
319+ def setUp(self):
320+ super(TestSlave, self).setUp()
321+ self.slave_helper = SlaveTestHelpers()
322+ self.slave_helper.setUp()
323+ self.addCleanup(self.slave_helper.cleanUp)
324+
325+ # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
326+ # BuilderSlave in buildd-slave.txt and in other places. The tests here
327+ # ought to become the canonical tests for BuilderSlave vs running buildd
328+ # XML-RPC server interaction.
329+
330 def test_abort(self):
331- slave = self.getClientSlave()
332+ slave = self.slave_helper.getClientSlave()
333 # We need to be in a BUILDING state before we can abort.
334- self.triggerGoodBuild(slave)
335+ self.slave_helper.triggerGoodBuild(slave)
336 result = slave.abort()
337 self.assertEqual(result, BuilderStatus.ABORTING)
338
339@@ -549,8 +575,8 @@
340 # valid chroot & filemaps works and returns a BuilderStatus of
341 # BUILDING.
342 build_id = 'some-id'
343- slave = self.getClientSlave()
344- result = self.triggerGoodBuild(slave, build_id)
345+ slave = self.slave_helper.getClientSlave()
346+ result = self.slave_helper.triggerGoodBuild(slave, build_id)
347 self.assertEqual([BuilderStatus.BUILDING, build_id], result)
348
349 def test_clean(self):
350@@ -564,15 +590,15 @@
351 def test_echo(self):
352 # Calling 'echo' contacts the server which returns the arguments we
353 # gave it.
354- self.getServerSlave()
355- slave = self.getClientSlave()
356+ self.slave_helper.getServerSlave()
357+ slave = self.slave_helper.getClientSlave()
358 result = slave.echo('foo', 'bar', 42)
359 self.assertEqual(['foo', 'bar', 42], result)
360
361 def test_info(self):
362 # Calling 'info' gets some information about the slave.
363- self.getServerSlave()
364- slave = self.getClientSlave()
365+ self.slave_helper.getServerSlave()
366+ slave = self.slave_helper.getClientSlave()
367 result = slave.info()
368 # We're testing the hard-coded values, since the version is hard-coded
369 # into the remote slave, the supported build managers are hard-coded
370@@ -588,17 +614,17 @@
371 def test_initial_status(self):
372 # Calling 'status' returns the current status of the slave. The
373 # initial status is IDLE.
374- self.getServerSlave()
375- slave = self.getClientSlave()
376+ self.slave_helper.getServerSlave()
377+ slave = self.slave_helper.getClientSlave()
378 status = slave.status()
379 self.assertEqual([BuilderStatus.IDLE, ''], status)
380
381 def test_status_after_build(self):
382 # Calling 'status' returns the current status of the slave. After a
383 # build has been triggered, the status is BUILDING.
384- slave = self.getClientSlave()
385+ slave = self.slave_helper.getClientSlave()
386 build_id = 'status-build-id'
387- self.triggerGoodBuild(slave, build_id)
388+ self.slave_helper.triggerGoodBuild(slave, build_id)
389 status = slave.status()
390 self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2])
391 [log_file] = status[2:]
392@@ -606,15 +632,84 @@
393
394 def test_ensurepresent_not_there(self):
395 # ensurepresent checks to see if a file is there.
396- self.getServerSlave()
397- slave = self.getClientSlave()
398- result = slave.ensurepresent('blahblah', None, None, None)
399- self.assertEqual([False, 'No URL'], result)
400+ self.slave_helper.getServerSlave()
401+ slave = self.slave_helper.getClientSlave()
402+ d = slave.ensurepresent('blahblah', None, None, None)
403+ d.addCallback(self.assertEqual, [False, 'No URL'])
404+ return d
405
406 def test_ensurepresent_actually_there(self):
407 # ensurepresent checks to see if a file is there.
408- tachandler = self.getServerSlave()
409- slave = self.getClientSlave()
410- self.makeCacheFile(tachandler, 'blahblah')
411- result = slave.ensurepresent('blahblah', None, None, None)
412- self.assertEqual([True, 'No URL'], result)
413+ tachandler = self.slave_helper.getServerSlave()
414+ slave = self.slave_helper.getClientSlave()
415+ self.slave_helper.makeCacheFile(tachandler, 'blahblah')
416+ d = slave.ensurepresent('blahblah', None, None, None)
417+ d.addCallback(self.assertEqual, [True, 'No URL'])
418+ return d
419+
420+ def test_sendFileToSlave_not_there(self):
421+ self.slave_helper.getServerSlave()
422+ slave = self.slave_helper.getClientSlave()
423+ d = slave.sendFileToSlave('blahblah', None, None, None)
424+ return self.assertFailure(d, CannotFetchFile)
425+
426+ def test_sendFileToSlave_actually_there(self):
427+ tachandler = self.slave_helper.getServerSlave()
428+ slave = self.slave_helper.getClientSlave()
429+ self.slave_helper.makeCacheFile(tachandler, 'blahblah')
430+ d = slave.sendFileToSlave('blahblah', None, None, None)
431+ def check_present(ignored):
432+ d = slave.ensurepresent('blahblah', None, None, None)
433+ return d.addCallback(self.assertEqual, [True, 'No URL'])
434+ d.addCallback(check_present)
435+ return d
436+
437+
438+class TestSlaveWithLibrarian(TrialTestCase):
439+ """Tests that need more of Launchpad to run."""
440+
441+ layer = TwistedLaunchpadZopelessLayer
442+
443+ def setUp(self):
444+ super(TestSlaveWithLibrarian, self)
445+ self.slave_helper = SlaveTestHelpers()
446+ self.slave_helper.setUp()
447+ self.addCleanup(self.slave_helper.cleanUp)
448+ self.factory = LaunchpadObjectFactory()
449+ login_as(ANONYMOUS)
450+ self.addCleanup(logout)
451+
452+ def test_ensurepresent_librarian(self):
453+ # ensurepresent, when given an http URL for a file will download the
454+ # file from that URL and report that the file is present, and it was
455+ # downloaded.
456+
457+ # Use the Librarian because it's a "convenient" web server.
458+ lf = self.factory.makeLibraryFileAlias(
459+ 'HelloWorld.txt', content="Hello World")
460+ self.layer.txn.commit()
461+ self.slave_helper.getServerSlave()
462+ slave = self.slave_helper.getClientSlave()
463+ d = slave.ensurepresent(
464+ lf.content.sha1, lf.http_url, "", "")
465+ d.addCallback(self.assertEqual, [True, 'Download'])
466+ return d
467+
468+ def test_retrieve_files_from_filecache(self):
469+ # Files that are present on the slave can be downloaded with a
470+ # filename made from the sha1 of the content underneath the
471+ # 'filecache' directory.
472+ content = "Hello World"
473+ lf = self.factory.makeLibraryFileAlias(
474+ 'HelloWorld.txt', content=content)
475+ self.layer.txn.commit()
476+ expected_url = '%s/filecache/%s' % (
477+ self.slave_helper.BASE_URL, lf.content.sha1)
478+ self.slave_helper.getServerSlave()
479+ slave = self.slave_helper.getClientSlave()
480+ d = slave.ensurepresent(
481+ lf.content.sha1, lf.http_url, "", "")
482+ def check_file(ignored):
483+ d = getPage(expected_url.encode('utf8'))
484+ return d.addCallback(self.assertEqual, content)
485+ return d.addCallback(check_file)
486
487=== modified file 'lib/lp/code/model/recipebuilder.py'
488--- lib/lp/code/model/recipebuilder.py 2010-08-20 20:31:18 +0000
489+++ lib/lp/code/model/recipebuilder.py 2010-09-22 11:33:47 +0000
490@@ -122,33 +122,36 @@
491 if chroot is None:
492 raise CannotBuild("Unable to find a chroot for %s" %
493 distroarchseries.displayname)
494- self._builder.slave.cacheFile(logger, chroot)
495-
496- # Generate a string which can be used to cross-check when obtaining
497- # results so we know we are referring to the right database object in
498- # subsequent runs.
499- buildid = "%s-%s" % (self.build.id, build_queue_id)
500- cookie = self.buildfarmjob.generateSlaveBuildCookie()
501- chroot_sha1 = chroot.content.sha1
502- logger.debug(
503- "Initiating build %s on %s" % (buildid, self._builder.url))
504-
505- args = self._extraBuildArgs(distroarchseries, logger)
506- status, info = self._builder.slave.build(
507- cookie, "sourcepackagerecipe", chroot_sha1, {}, args)
508- message = """%s (%s):
509- ***** RESULT *****
510- %s
511- %s: %s
512- ******************
513- """ % (
514- self._builder.name,
515- self._builder.url,
516- args,
517- status,
518- info,
519- )
520- logger.info(message)
521+ d = self._builder.slave.cacheFile(logger, chroot)
522+
523+ def got_cache_file(ignored):
524+ # Generate a string which can be used to cross-check when obtaining
525+ # results so we know we are referring to the right database object in
526+ # subsequent runs.
527+ buildid = "%s-%s" % (self.build.id, build_queue_id)
528+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
529+ chroot_sha1 = chroot.content.sha1
530+ logger.debug(
531+ "Initiating build %s on %s" % (buildid, self._builder.url))
532+
533+ args = self._extraBuildArgs(distroarchseries, logger)
534+ # XXX: Soon to be async
535+ status, info = self._builder.slave.build(
536+ cookie, "sourcepackagerecipe", chroot_sha1, {}, args)
537+ message = """%s (%s):
538+ ***** RESULT *****
539+ %s
540+ %s: %s
541+ ******************
542+ """ % (
543+ self._builder.name,
544+ self._builder.url,
545+ args,
546+ status,
547+ info,
548+ )
549+ logger.info(message)
550+ return d.addCallback(got_cache_file)
551
552 def verifyBuildRequest(self, logger):
553 """Assert some pre-build checks.
554
555=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
556--- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-22 11:33:46 +0000
557+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-22 11:33:47 +0000
558@@ -11,6 +11,7 @@
559 'BinaryPackageBuildBehavior',
560 ]
561
562+from twisted.internet import defer
563 from zope.interface import implements
564
565 from canonical.launchpad.webapp import urlappend
566@@ -38,56 +39,66 @@
567 logger.info("startBuild(%s, %s, %s, %s)", self._builder.url,
568 spr.name, spr.version, self.build.pocket.title)
569
570- def dispatchBuildToSlave(self, build_queue_id, logger):
571- """See `IBuildFarmJobBehavior`."""
572-
573- # Start the binary package build on the slave builder. First
574- # we send the chroot.
575- chroot = self.build.distro_arch_series.getChroot()
576- self._builder.slave.cacheFile(logger, chroot)
577-
578+ def _buildFilemapStructure(self, ignored, logger):
579 # Build filemap structure with the files required in this build
580 # and send them to the slave.
581 # If the build is private we tell the slave to get the files from the
582 # archive instead of the librarian because the slaves cannot
583 # access the restricted librarian.
584+ dl = []
585 private = self.build.archive.private
586 if private:
587- self._cachePrivateSourceOnSlave(logger)
588+ dl.extend(self._cachePrivateSourceOnSlave(logger))
589 filemap = {}
590 for source_file in self.build.source_package_release.files:
591 lfa = source_file.libraryfile
592 filemap[lfa.filename] = lfa.content.sha1
593 if not private:
594- self._builder.slave.cacheFile(logger, source_file.libraryfile)
595-
596- # Generate a string which can be used to cross-check when obtaining
597- # results so we know we are referring to the right database object in
598- # subsequent runs.
599- buildid = "%s-%s" % (self.build.id, build_queue_id)
600- cookie = self.buildfarmjob.generateSlaveBuildCookie()
601- chroot_sha1 = chroot.content.sha1
602- logger.debug(
603- "Initiating build %s on %s" % (buildid, self._builder.url))
604-
605- args = self._extraBuildArgs(self.build)
606- status, info = self._builder.slave.build(
607- cookie, "binarypackage", chroot_sha1, filemap, args)
608- message = """%s (%s):
609- ***** RESULT *****
610- %s
611- %s
612- %s: %s
613- ******************
614- """ % (
615- self._builder.name,
616- self._builder.url,
617- filemap,
618- args,
619- status,
620- info,
621- )
622- logger.info(message)
623+ dl.append(
624+ self._builder.slave.cacheFile(
625+ logger, source_file.libraryfile))
626+ d = defer.gatherResults(dl)
627+ return d.addCallback(lambda ignored: filemap)
628+
629+ def dispatchBuildToSlave(self, build_queue_id, logger):
630+ """See `IBuildFarmJobBehavior`."""
631+
632+ # Start the binary package build on the slave builder. First
633+ # we send the chroot.
634+ chroot = self.build.distro_arch_series.getChroot()
635+ d = self._builder.slave.cacheFile(logger, chroot)
636+ d.addCallback(self._buildFilemapStructure, logger)
637+
638+ def got_filemap(filemap):
639+ # Generate a string which can be used to cross-check when obtaining
640+ # results so we know we are referring to the right database object in
641+ # subsequent runs.
642+ buildid = "%s-%s" % (self.build.id, build_queue_id)
643+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
644+ chroot_sha1 = chroot.content.sha1
645+ logger.debug(
646+ "Initiating build %s on %s" % (buildid, self._builder.url))
647+
648+ args = self._extraBuildArgs(self.build)
649+ status, info = self._builder.slave.build(
650+ cookie, "binarypackage", chroot_sha1, filemap, args)
651+ message = """%s (%s):
652+ ***** RESULT *****
653+ %s
654+ %s
655+ %s: %s
656+ ******************
657+ """ % (
658+ self._builder.name,
659+ self._builder.url,
660+ filemap,
661+ args,
662+ status,
663+ info,
664+ )
665+ logger.info(message)
666+
667+ return d.addCallback(got_filemap)
668
669 def verifyBuildRequest(self, logger):
670 """Assert some pre-build checks.
671@@ -154,6 +165,8 @@
672 """Ask the slave to download source files for a private build.
673
674 :param logger: A logger used for providing debug information.
675+ :return: A list of Deferreds, each of which represents a request
676+ to cache a file.
677 """
678 # The URL to the file in the archive consists of these parts:
679 # archive_url / makePoolPath() / filename
680@@ -165,6 +178,7 @@
681 archive = self.build.archive
682 archive_url = archive.archive_url
683 component_name = self.build.current_component.name
684+ dl = []
685 for source_file in self.build.source_package_release.files:
686 file_name = source_file.libraryfile.filename
687 sha1 = source_file.libraryfile.content.sha1
688@@ -175,8 +189,10 @@
689 logger.debug("Asking builder on %s to ensure it has file %s "
690 "(%s, %s)" % (
691 self._builder.url, file_name, url, sha1))
692- self._builder.slave.sendFileToSlave(
693- sha1, url, "buildd", archive.buildd_secret)
694+ dl.append(
695+ self._builder.slave.sendFileToSlave(
696+ sha1, url, "buildd", archive.buildd_secret))
697+ return dl
698
699 def _extraBuildArgs(self, build):
700 """
701
702=== modified file 'lib/lp/soyuz/tests/soyuzbuilddhelpers.py'
703--- lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-22 11:33:46 +0000
704+++ lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-22 11:33:47 +0000
705@@ -19,6 +19,8 @@
706 from StringIO import StringIO
707 import xmlrpclib
708
709+from twisted.internet import defer
710+
711 from lp.buildmaster.interfaces.builder import CannotFetchFile
712 from lp.buildmaster.model.builder import (
713 rescueBuilderIfLost,
714@@ -99,7 +101,7 @@
715
716 def ensurepresent(self, sha1, url, user=None, password=None):
717 self.call_log.append(('ensurepresent', url, user, password))
718- return True, None
719+ return defer.succeed((True, None))
720
721 def build(self, buildid, buildtype, chroot, filemap, args):
722 self.call_log.append(
723@@ -123,9 +125,11 @@
724
725 def sendFileToSlave(self, sha1, url, username="", password=""):
726 self.call_log.append('sendFileToSlave')
727- present, info = self.ensurepresent(sha1, url, username, password)
728- if not present:
729- raise CannotFetchFile(url, info)
730+ d = self.ensurepresent(sha1, url, username, password)
731+ def check_present((present, info)):
732+ if not present:
733+ raise CannotFetchFile(url, info)
734+ return d.addCallback(check_present)
735
736 def cacheFile(self, logger, libraryfilealias):
737 self.call_log.append('cacheFile')
738
739=== modified file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py'
740--- lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-08-20 20:31:18 +0000
741+++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-09-22 11:33:47 +0000
742@@ -41,16 +41,18 @@
743 """See `IBuildFarmJobBehavior`."""
744 chroot = self._getChroot()
745 chroot_sha1 = chroot.content.sha1
746- self._builder.slave.cacheFile(logger, chroot)
747- cookie = self.buildfarmjob.generateSlaveBuildCookie()
748-
749- args = {'arch_tag': self._getDistroArchSeries().architecturetag}
750- args.update(self.buildfarmjob.metadata)
751-
752- filemap = {}
753-
754- self._builder.slave.build(
755- cookie, self.build_type, chroot_sha1, filemap, args)
756+ d = self._builder.slave.cacheFile(logger, chroot)
757+ def got_cache_file(ignored):
758+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
759+
760+ args = {'arch_tag': self._getDistroArchSeries().architecturetag}
761+ args.update(self.buildfarmjob.metadata)
762+
763+ filemap = {}
764+
765+ return self._builder.slave.build(
766+ cookie, self.build_type, chroot_sha1, filemap, args)
767+ return d.addCallback(got_cache_file)
768
769 def _getChroot(self):
770 return self._getDistroArchSeries().getChroot()