Launchpad itself

Merge lp:~jml/launchpad/buildd-deferred-fo-shizzle into lp:launchpad

  1. buildd-deferred-fo-shizzle
  2. Merge into devel
Proposed by Jonathan Lange
Status: Merged
Merged at revision: 11801
Proposed branch: lp:~jml/launchpad/buildd-deferred-fo-shizzle
Merge into: lp:launchpad
Prerequisite: lp:~jml/launchpad/buildd-deferred
Diff against target: 816 lines (+305/-193)
9 files modified
lib/lp/buildmaster/manager.py (+25/-19)
lib/lp/buildmaster/model/builder.py (+38/-17)
lib/lp/buildmaster/tests/test_builder.py (+138/-43)
lib/lp/code/model/recipebuilder.py (+30/-27)
lib/lp/soyuz/doc/buildd-slave.txt (+1/-37)
lib/lp/soyuz/model/binarypackagebuildbehavior.py (+57/-40)
lib/lp/soyuz/scripts/buildd.py (+2/-0)
lib/lp/soyuz/tests/soyuzbuilddhelpers.py (+2/-0)
lib/lp/translations/model/translationtemplatesbuildbehavior.py (+12/-10)
To merge this branch: bzr merge lp:~jml/launchpad/buildd-deferred-fo-shizzle
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+36037@code.launchpad.net

Description of the change

Actually starting to make BuilderSlave have asynchronous APIs.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/manager.py'
2--- lib/lp/buildmaster/manager.py 2010-09-20 10:21:32 +0000
3+++ lib/lp/buildmaster/manager.py 2010-09-21 09:54:50 +0000
4@@ -283,15 +283,15 @@
5 """Scan the builder and dispatch to it or deal with failures."""
6 self.logger.debug("Scanning builder: %s" % self.builder_name)
7
8- try:
9- slave = self.scan()
10+ d = self.scan()
11+
12+ def got_slave(slave):
13 if slave is None:
14- self.scheduleNextScanCycle()
15+ return self.scheduleNextScanCycle()
16 else:
17- # XXX: Ought to return Deferred.
18- self.resumeAndDispatch(slave)
19- except:
20- error = Failure()
21+ return self.resumeAndDispatch(slave)
22+
23+ def disaster(error):
24 self.logger.info("Scanning failed with: %s\n%s" %
25 (error.getErrorMessage(), error.getTraceback()))
26
27@@ -307,7 +307,11 @@
28 assessFailureCounts(builder, error.getErrorMessage())
29 transaction.commit()
30
31- self.scheduleNextScanCycle()
32+ return self.scheduleNextScanCycle()
33+
34+ d.addCallback(got_slave)
35+ d.addErrback(disaster)
36+ return d
37
38 @write_transaction
39 def scan(self):
40@@ -346,7 +350,7 @@
41 if self.builder.manual:
42 self.logger.debug(
43 '%s is in manual mode, not dispatching.' % self.builder.name)
44- return None
45+ return defer.succeed(None)
46
47 # If the builder is marked unavailable, don't dispatch anything.
48 # Additionaly, because builders can be removed from the pool at
49@@ -362,7 +366,7 @@
50 "job" % self.builder.name)
51 job.reset()
52 transaction.commit()
53- return None
54+ return defer.succeed(None)
55
56 # See if there is a job we can dispatch to the builder slave.
57
58@@ -374,15 +378,17 @@
59 self.builder.name, self.builder.url, self.builder.vm_host)
60 # XXX: Passing buildd_slave=slave overwrites the 'slave' property of
61 # self.builder. Not sure why this is needed yet.
62- self.builder.findAndStartJob(buildd_slave=slave)
63- if self.builder.currentjob is not None:
64- # After a successful dispatch we can reset the
65- # failure_count.
66- self.builder.resetFailureCount()
67- transaction.commit()
68- return slave
69-
70- return None
71+ d = self.builder.findAndStartJob(buildd_slave=slave)
72+ def job_started(candidate):
73+ if self.builder.currentjob is not None:
74+ # After a successful dispatch we can reset the
75+ # failure_count.
76+ self.builder.resetFailureCount()
77+ transaction.commit()
78+ return slave
79+ else:
80+ return None
81+ return d.addCallback(job_started)
82
83 def resumeAndDispatch(self, slave):
84 """Chain the resume and dispatching Deferreds."""
85
86=== modified file 'lib/lp/buildmaster/model/builder.py'
87--- lib/lp/buildmaster/model/builder.py 2010-09-21 09:05:34 +0000
88+++ lib/lp/buildmaster/model/builder.py 2010-09-21 09:54:50 +0000
89@@ -34,6 +34,7 @@
90 Count,
91 Sum,
92 )
93+from twisted.internet import defer
94 from zope.component import getUtility
95 from zope.interface import implements
96
97@@ -161,8 +162,10 @@
98 return self._server.status()
99
100 def ensurepresent(self, sha1sum, url, username, password):
101+ # XXX: Nothing external calls this. Make it private.
102 """Attempt to ensure the given file is present."""
103- return self._server.ensurepresent(sha1sum, url, username, password)
104+ return defer.succeed(
105+ self._server.ensurepresent(sha1sum, url, username, password))
106
107 def getFile(self, sha_sum):
108 """Construct a file-like object to return the named file."""
109@@ -201,13 +204,15 @@
110 logger.debug("Asking builder on %s to ensure it has file %s "
111 "(%s, %s)" % (self.urlbase, libraryfilealias.filename,
112 url, libraryfilealias.content.sha1))
113- self.sendFileToSlave(libraryfilealias.content.sha1, url)
114+ return self.sendFileToSlave(libraryfilealias.content.sha1, url)
115
116 def sendFileToSlave(self, sha1, url, username="", password=""):
117 """Helper to send the file at 'url' with 'sha1' to this builder."""
118- present, info = self.ensurepresent(sha1, url, username, password)
119- if not present:
120- raise CannotFetchFile(url, info)
121+ d = self.ensurepresent(sha1, url, username, password)
122+ def check_present((present, info)):
123+ if not present:
124+ raise CannotFetchFile(url, info)
125+ return d.addCallback(check_present)
126
127 def build(self, buildid, builder_type, chroot_sha1, filemap, args):
128 """Build a thing on this build slave.
129@@ -464,14 +469,19 @@
130
131 # Do it.
132 build_queue_item.markAsBuilding(self)
133- try:
134- self.current_build_behavior.dispatchBuildToSlave(
135- build_queue_item.id, logger)
136- except BuildSlaveFailure, e:
137- logger.debug("Disabling builder: %s" % self.url, exc_info=1)
138+
139+ d = self.current_build_behavior.dispatchBuildToSlave(
140+ build_queue_item.id, logger)
141+
142+ def eb_slave_failure(failure):
143+ failure.trap(BuildSlaveFailure)
144+ e = failure.value
145 self.failBuilder(
146 "Exception (%s) when setting up to new job" % (e,))
147- except CannotFetchFile, e:
148+
149+ def eb_cannot_fetch_file(failure):
150+ failure.trap(CannotFetchFile)
151+ e = failure.value
152 message = """Slave '%s' (%s) was unable to fetch file.
153 ****** URL ********
154 %s
155@@ -480,11 +490,19 @@
156 *******************
157 """ % (self.name, self.url, e.file_url, e.error_information)
158 raise BuildDaemonError(message)
159- except socket.error, e:
160+
161+ def eb_socket_error(failure):
162+ failure.trap(socket.error)
163+ e = failure.value
164 error_message = "Exception (%s) when setting up new job" % (e,)
165 self.handleTimeout(logger, error_message)
166 raise BuildSlaveFailure
167
168+ d.addErrback(eb_slave_failure)
169+ d.addErrback(eb_cannot_fetch_file)
170+ d.addErrback(eb_socket_error)
171+ return d
172+
173 def failBuilder(self, reason):
174 """See IBuilder"""
175 # XXX cprov 2007-04-17: ideally we should be able to notify the
176@@ -677,10 +695,13 @@
177 :param candidate: The job to dispatch.
178 """
179 logger = self._getSlaveScannerLogger()
180- try:
181- self.startBuild(candidate, logger)
182- except (BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch), err:
183+ d = self.startBuild(candidate, logger)
184+ def warn_on_error(failure):
185+ failure.trap(
186+ BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch)
187+ err = failure.value
188 logger.warn('Could not build: %s' % err)
189+ return d.addErrback(warn_on_error)
190
191 def handleTimeout(self, logger, error_message):
192 """See IBuilder."""
193@@ -721,8 +742,8 @@
194 if buildd_slave is not None:
195 self.setSlaveForTesting(buildd_slave)
196
197- self._dispatchBuildCandidate(candidate)
198- return candidate
199+ d = self._dispatchBuildCandidate(candidate)
200+ return d.addCallback(lambda ignored: candidate)
201
202 def getBuildQueue(self):
203 """See `IBuilder`."""
204
205=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
206--- lib/lp/buildmaster/tests/test_builder.py 2010-09-21 09:05:34 +0000
207+++ lib/lp/buildmaster/tests/test_builder.py 2010-09-21 09:54:50 +0000
208@@ -8,8 +8,10 @@
209 import socket
210 import xmlrpclib
211
212-from testtools.content import Content
213-from testtools.content_type import UTF8_TEXT
214+import fixtures
215+
216+from twisted.trial.unittest import TestCase as TrialTestCase
217+from twisted.web.client import getPage
218
219 from zope.component import getUtility
220 from zope.security.proxy import removeSecurityProxy
221@@ -24,10 +26,16 @@
222 )
223 from canonical.testing.layers import (
224 DatabaseFunctionalLayer,
225- LaunchpadZopelessLayer
226+ LaunchpadZopelessLayer,
227+ TwistedLaunchpadZopelessLayer,
228+ TwistedLayer,
229 )
230 from lp.buildmaster.enums import BuildStatus
231-from lp.buildmaster.interfaces.builder import IBuilder, IBuilderSet
232+from lp.buildmaster.interfaces.builder import (
233+ CannotFetchFile,
234+ IBuilder,
235+ IBuilderSet,
236+ )
237 from lp.buildmaster.interfaces.buildfarmjobbehavior import (
238 IBuildFarmJobBehavior,
239 )
240@@ -49,9 +57,12 @@
241 )
242 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
243 from lp.testing import (
244- TestCase,
245+ ANONYMOUS,
246+ login_as,
247+ logout,
248 TestCaseWithFactory,
249 )
250+from lp.testing.factory import LaunchpadObjectFactory
251 from lp.testing.fakemethod import FakeMethod
252
253
254@@ -467,19 +478,11 @@
255 self.builder.current_build_behavior, BinaryPackageBuildBehavior)
256
257
258-class TestSlave(TestCase):
259- """
260- Integration tests for BuilderSlave that verify how it works against a
261- real slave server.
262- """
263-
264- # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
265- # BuilderSlave in buildd-slave.txt and in other places. The tests here
266- # ought to become the canonical tests for BuilderSlave vs running buildd
267- # XML-RPC server interaction.
268+class SlaveTestHelpers(fixtures.Fixture):
269
270 # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`.
271- TEST_URL = 'http://localhost:8221/rpc/'
272+ BASE_URL = 'http://localhost:8221'
273+ TEST_URL = '%s/rpc/' % (BASE_URL,)
274
275 def getServerSlave(self):
276 """Set up a test build slave server.
277@@ -488,11 +491,14 @@
278 """
279 tachandler = BuilddSlaveTestSetup()
280 tachandler.setUp()
281- def addLogFile(exc_info):
282- self.addDetail(
283- 'xmlrpc-log-file',
284- Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))
285- self.addOnException(addLogFile)
286+ # Basically impossible to do this w/ TrialTestCase. But it would be
287+ # really nice to keep it.
288+ #
289+ # def addLogFile(exc_info):
290+ # self.addDetail(
291+ # 'xmlrpc-log-file',
292+ # Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))
293+ # self.addOnException(addLogFile)
294 self.addCleanup(tachandler.tearDown)
295 return tachandler
296
297@@ -527,7 +533,7 @@
298 :return: The build id returned by the slave.
299 """
300 if build_id is None:
301- build_id = self.getUniqueString()
302+ build_id = 'random-build-id'
303 tachandler = self.getServerSlave()
304 chroot_file = 'fake-chroot'
305 dsc_file = 'thing'
306@@ -537,10 +543,30 @@
307 build_id, 'debian', chroot_file, {'.dsc': dsc_file},
308 {'ogrecomponent': 'main'})
309
310+
311+class TestSlave(TrialTestCase):
312+ """
313+ Integration tests for BuilderSlave that verify how it works against a
314+ real slave server.
315+ """
316+
317+ layer = TwistedLayer
318+
319+ def setUp(self):
320+ super(TestSlave, self).setUp()
321+ self.slave_helper = SlaveTestHelpers()
322+ self.slave_helper.setUp()
323+ self.addCleanup(self.slave_helper.cleanUp)
324+
325+ # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
326+ # BuilderSlave in buildd-slave.txt and in other places. The tests here
327+ # ought to become the canonical tests for BuilderSlave vs running buildd
328+ # XML-RPC server interaction.
329+
330 def test_abort(self):
331- slave = self.getClientSlave()
332+ slave = self.slave_helper.getClientSlave()
333 # We need to be in a BUILDING state before we can abort.
334- self.triggerGoodBuild(slave)
335+ self.slave_helper.triggerGoodBuild(slave)
336 result = slave.abort()
337 self.assertEqual(result, BuilderStatus.ABORTING)
338
339@@ -549,8 +575,8 @@
340 # valid chroot & filemaps works and returns a BuilderStatus of
341 # BUILDING.
342 build_id = 'some-id'
343- slave = self.getClientSlave()
344- result = self.triggerGoodBuild(slave, build_id)
345+ slave = self.slave_helper.getClientSlave()
346+ result = self.slave_helper.triggerGoodBuild(slave, build_id)
347 self.assertEqual([BuilderStatus.BUILDING, build_id], result)
348
349 def test_clean(self):
350@@ -564,15 +590,15 @@
351 def test_echo(self):
352 # Calling 'echo' contacts the server which returns the arguments we
353 # gave it.
354- self.getServerSlave()
355- slave = self.getClientSlave()
356+ self.slave_helper.getServerSlave()
357+ slave = self.slave_helper.getClientSlave()
358 result = slave.echo('foo', 'bar', 42)
359 self.assertEqual(['foo', 'bar', 42], result)
360
361 def test_info(self):
362 # Calling 'info' gets some information about the slave.
363- self.getServerSlave()
364- slave = self.getClientSlave()
365+ self.slave_helper.getServerSlave()
366+ slave = self.slave_helper.getClientSlave()
367 result = slave.info()
368 # We're testing the hard-coded values, since the version is hard-coded
369 # into the remote slave, the supported build managers are hard-coded
370@@ -588,17 +614,17 @@
371 def test_initial_status(self):
372 # Calling 'status' returns the current status of the slave. The
373 # initial status is IDLE.
374- self.getServerSlave()
375- slave = self.getClientSlave()
376+ self.slave_helper.getServerSlave()
377+ slave = self.slave_helper.getClientSlave()
378 status = slave.status()
379 self.assertEqual([BuilderStatus.IDLE, ''], status)
380
381 def test_status_after_build(self):
382 # Calling 'status' returns the current status of the slave. After a
383 # build has been triggered, the status is BUILDING.
384- slave = self.getClientSlave()
385+ slave = self.slave_helper.getClientSlave()
386 build_id = 'status-build-id'
387- self.triggerGoodBuild(slave, build_id)
388+ self.slave_helper.triggerGoodBuild(slave, build_id)
389 status = slave.status()
390 self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2])
391 [log_file] = status[2:]
392@@ -606,15 +632,84 @@
393
394 def test_ensurepresent_not_there(self):
395 # ensurepresent checks to see if a file is there.
396- self.getServerSlave()
397- slave = self.getClientSlave()
398- result = slave.ensurepresent('blahblah', None, None, None)
399- self.assertEqual([False, 'No URL'], result)
400+ self.slave_helper.getServerSlave()
401+ slave = self.slave_helper.getClientSlave()
402+ d = slave.ensurepresent('blahblah', None, None, None)
403+ d.addCallback(self.assertEqual, [False, 'No URL'])
404+ return d
405
406 def test_ensurepresent_actually_there(self):
407 # ensurepresent checks to see if a file is there.
408- tachandler = self.getServerSlave()
409- slave = self.getClientSlave()
410- self.makeCacheFile(tachandler, 'blahblah')
411- result = slave.ensurepresent('blahblah', None, None, None)
412- self.assertEqual([True, 'No URL'], result)
413+ tachandler = self.slave_helper.getServerSlave()
414+ slave = self.slave_helper.getClientSlave()
415+ self.slave_helper.makeCacheFile(tachandler, 'blahblah')
416+ d = slave.ensurepresent('blahblah', None, None, None)
417+ d.addCallback(self.assertEqual, [True, 'No URL'])
418+ return d
419+
420+ def test_sendFileToSlave_not_there(self):
421+ self.slave_helper.getServerSlave()
422+ slave = self.slave_helper.getClientSlave()
423+ d = slave.sendFileToSlave('blahblah', None, None, None)
424+ return self.assertFailure(d, CannotFetchFile)
425+
426+ def test_sendFileToSlave_actually_there(self):
427+ tachandler = self.slave_helper.getServerSlave()
428+ slave = self.slave_helper.getClientSlave()
429+ self.slave_helper.makeCacheFile(tachandler, 'blahblah')
430+ d = slave.sendFileToSlave('blahblah', None, None, None)
431+ def check_present(ignored):
432+ d = slave.ensurepresent('blahblah', None, None, None)
433+ return d.addCallback(self.assertEqual, [True, 'No URL'])
434+ d.addCallback(check_present)
435+ return d
436+
437+
438+class TestSlaveWithLibrarian(TrialTestCase):
439+ """Tests that need more of Launchpad to run."""
440+
441+ layer = TwistedLaunchpadZopelessLayer
442+
443+ def setUp(self):
444+ super(TestSlaveWithLibrarian, self)
445+ self.slave_helper = SlaveTestHelpers()
446+ self.slave_helper.setUp()
447+ self.addCleanup(self.slave_helper.cleanUp)
448+ self.factory = LaunchpadObjectFactory()
449+ login_as(ANONYMOUS)
450+ self.addCleanup(logout)
451+
452+ def test_ensurepresent_librarian(self):
453+ # ensurepresent, when given an http URL for a file will download the
454+ # file from that URL and report that the file is present, and it was
455+ # downloaded.
456+
457+ # Use the Librarian because it's a "convenient" web server.
458+ lf = self.factory.makeLibraryFileAlias(
459+ 'HelloWorld.txt', content="Hello World")
460+ self.layer.txn.commit()
461+ self.slave_helper.getServerSlave()
462+ slave = self.slave_helper.getClientSlave()
463+ d = slave.ensurepresent(
464+ lf.content.sha1, lf.http_url, "", "")
465+ d.addCallback(self.assertEqual, [True, 'Download'])
466+ return d
467+
468+ def test_retrieve_files_from_filecache(self):
469+ # Files that are present on the slave can be downloaded with a
470+ # filename made from the sha1 of the content underneath the
471+ # 'filecache' directory.
472+ content = "Hello World"
473+ lf = self.factory.makeLibraryFileAlias(
474+ 'HelloWorld.txt', content=content)
475+ self.layer.txn.commit()
476+ expected_url = '%s/filecache/%s' % (
477+ self.slave_helper.BASE_URL, lf.content.sha1)
478+ self.slave_helper.getServerSlave()
479+ slave = self.slave_helper.getClientSlave()
480+ d = slave.ensurepresent(
481+ lf.content.sha1, lf.http_url, "", "")
482+ def check_file(ignored):
483+ d = getPage(expected_url.encode('utf8'))
484+ return d.addCallback(self.assertEqual, content)
485+ return d.addCallback(check_file)
486
487=== modified file 'lib/lp/code/model/recipebuilder.py'
488--- lib/lp/code/model/recipebuilder.py 2010-08-20 20:31:18 +0000
489+++ lib/lp/code/model/recipebuilder.py 2010-09-21 09:54:50 +0000
490@@ -122,33 +122,36 @@
491 if chroot is None:
492 raise CannotBuild("Unable to find a chroot for %s" %
493 distroarchseries.displayname)
494- self._builder.slave.cacheFile(logger, chroot)
495-
496- # Generate a string which can be used to cross-check when obtaining
497- # results so we know we are referring to the right database object in
498- # subsequent runs.
499- buildid = "%s-%s" % (self.build.id, build_queue_id)
500- cookie = self.buildfarmjob.generateSlaveBuildCookie()
501- chroot_sha1 = chroot.content.sha1
502- logger.debug(
503- "Initiating build %s on %s" % (buildid, self._builder.url))
504-
505- args = self._extraBuildArgs(distroarchseries, logger)
506- status, info = self._builder.slave.build(
507- cookie, "sourcepackagerecipe", chroot_sha1, {}, args)
508- message = """%s (%s):
509- ***** RESULT *****
510- %s
511- %s: %s
512- ******************
513- """ % (
514- self._builder.name,
515- self._builder.url,
516- args,
517- status,
518- info,
519- )
520- logger.info(message)
521+ d = self._builder.slave.cacheFile(logger, chroot)
522+
523+ def got_cache_file(ignored):
524+ # Generate a string which can be used to cross-check when obtaining
525+ # results so we know we are referring to the right database object in
526+ # subsequent runs.
527+ buildid = "%s-%s" % (self.build.id, build_queue_id)
528+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
529+ chroot_sha1 = chroot.content.sha1
530+ logger.debug(
531+ "Initiating build %s on %s" % (buildid, self._builder.url))
532+
533+ args = self._extraBuildArgs(distroarchseries, logger)
534+ # XXX: Soon to be async
535+ status, info = self._builder.slave.build(
536+ cookie, "sourcepackagerecipe", chroot_sha1, {}, args)
537+ message = """%s (%s):
538+ ***** RESULT *****
539+ %s
540+ %s: %s
541+ ******************
542+ """ % (
543+ self._builder.name,
544+ self._builder.url,
545+ args,
546+ status,
547+ info,
548+ )
549+ logger.info(message)
550+ return d.addCallback(got_cache_file)
551
552 def verifyBuildRequest(self, logger):
553 """Assert some pre-build checks.
554
555=== modified file 'lib/lp/soyuz/doc/buildd-slave.txt'
556--- lib/lp/soyuz/doc/buildd-slave.txt 2010-04-30 10:00:34 +0000
557+++ lib/lp/soyuz/doc/buildd-slave.txt 2010-09-21 09:54:50 +0000
558@@ -13,43 +13,6 @@
559 >>> from canonical.buildd.tests import BuilddSlaveTestSetup
560 >>> BuilddSlaveTestSetup().setUp()
561
562-Use simple xmlrpclib client to certify the BuildSlave is running
563-
564- >>> import xmlrpclib
565- >>> slave = xmlrpclib.Server('http://localhost:8221/rpc/')
566- >>> slave.echo('Hello World')
567- ['Hello World']
568-
569-With slave protocol v1.0new, the only way to get files to the slave is to
570-put them in the librarian first...
571-
572- >>> from canonical.librarian.client import LibrarianClient
573- >>> from StringIO import StringIO
574- >>> from canonical.launchpad.database import LibraryFileAlias
575- >>> import transaction
576- >>> lc = LibrarianClient()
577- >>> helloworld = "Hello World"
578- >>> hw_sio = StringIO(helloworld)
579- >>> alias = lc.addFile("HelloWorld.txt", len(helloworld),
580- ... hw_sio, "text/plain")
581- >>> transaction.commit()
582- >>> lf = LibraryFileAlias.get(alias)
583- >>> present, info = slave.ensurepresent(
584- ... lf.content.sha1, lf.http_url, "", "")
585- >>> present, info
586- (True, 'Download')
587-
588-As of slave protocol v1.0new, /filecache/SHA1SUM is *THE* way
589-to retrieve files from the slave. Verify it works...
590-
591- >>> from urllib2 import urlopen
592- >>> f = urlopen("http://localhost:8221/filecache/" + lf.content.sha1)
593- >>> hw_str = f.read()
594- >>> f.close()
595- >>> hw_str == helloworld
596- True
597-
598-
599 == BuilderSet polling operations ==
600
601 >>> import logging
602@@ -90,6 +53,7 @@
603
604 At this point the buildd-slave is not accessible anymore.
605
606+ >>> import xmlrpclib
607 >>> s = xmlrpclib.Server('http://localhost:8221/rpc/')
608 >>> s.info()
609 Traceback (most recent call last):
610
611=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
612--- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-08-23 16:51:11 +0000
613+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-21 09:54:50 +0000
614@@ -11,6 +11,7 @@
615 'BinaryPackageBuildBehavior',
616 ]
617
618+from twisted.internet import defer
619 from zope.interface import implements
620
621 from canonical.launchpad.webapp import urlappend
622@@ -38,56 +39,67 @@
623 logger.info("startBuild(%s, %s, %s, %s)", self._builder.url,
624 spr.name, spr.version, self.build.pocket.title)
625
626- def dispatchBuildToSlave(self, build_queue_id, logger):
627- """See `IBuildFarmJobBehavior`."""
628-
629- # Start the binary package build on the slave builder. First
630- # we send the chroot.
631- chroot = self.build.distro_arch_series.getChroot()
632- self._builder.slave.cacheFile(logger, chroot)
633-
634+ def _buildFilemapStructure(self, logger):
635 # Build filemap structure with the files required in this build
636 # and send them to the slave.
637 # If the build is private we tell the slave to get the files from the
638 # archive instead of the librarian because the slaves cannot
639 # access the restricted librarian.
640+ dl = []
641 private = self.build.archive.private
642 if private:
643- self._cachePrivateSourceOnSlave(logger)
644+ dl.extend(self._cachePrivateSourceOnSlave(logger))
645 filemap = {}
646 for source_file in self.build.source_package_release.files:
647 lfa = source_file.libraryfile
648 filemap[lfa.filename] = lfa.content.sha1
649 if not private:
650- self._builder.slave.cacheFile(logger, source_file.libraryfile)
651-
652- # Generate a string which can be used to cross-check when obtaining
653- # results so we know we are referring to the right database object in
654- # subsequent runs.
655- buildid = "%s-%s" % (self.build.id, build_queue_id)
656- cookie = self.buildfarmjob.generateSlaveBuildCookie()
657- chroot_sha1 = chroot.content.sha1
658- logger.debug(
659- "Initiating build %s on %s" % (buildid, self._builder.url))
660-
661- args = self._extraBuildArgs(self.build)
662- status, info = self._builder.slave.build(
663- cookie, "binarypackage", chroot_sha1, filemap, args)
664- message = """%s (%s):
665- ***** RESULT *****
666- %s
667- %s
668- %s: %s
669- ******************
670- """ % (
671- self._builder.name,
672- self._builder.url,
673- filemap,
674- args,
675- status,
676- info,
677- )
678- logger.info(message)
679+ dl.append(
680+ self._builder.slave.cacheFile(
681+ logger, source_file.libraryfile))
682+ d = defer.gatherResults(dl)
683+ return d.addCallback(lambda ignored: filemap)
684+
685+ def dispatchBuildToSlave(self, build_queue_id, logger):
686+ """See `IBuildFarmJobBehavior`."""
687+
688+ # Start the binary package build on the slave builder. First
689+ # we send the chroot.
690+ chroot = self.build.distro_arch_series.getChroot()
691+ self._builder.slave.cacheFile(logger, chroot)
692+
693+ d = self._buildFilemapStructure(logger)
694+
695+ def got_filemap(filemap):
696+ # Generate a string which can be used to cross-check when
697+ # obtaining results so we know we are referring to the right
698+ # database object in subsequent runs.
699+ buildid = "%s-%s" % (self.build.id, build_queue_id)
700+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
701+ chroot_sha1 = chroot.content.sha1
702+ logger.debug(
703+ "Initiating build %s on %s" % (buildid, self._builder.url))
704+
705+ args = self._extraBuildArgs(self.build)
706+ status, info = self._builder.slave.build(
707+ cookie, "binarypackage", chroot_sha1, filemap, args)
708+ message = """%s (%s):
709+ ***** RESULT *****
710+ %s
711+ %s
712+ %s: %s
713+ ******************
714+ """ % (
715+ self._builder.name,
716+ self._builder.url,
717+ filemap,
718+ args,
719+ status,
720+ info,
721+ )
722+ logger.info(message)
723+
724+ return d.addCallback(got_filemap)
725
726 def verifyBuildRequest(self, logger):
727 """Assert some pre-build checks.
728@@ -153,6 +165,8 @@
729 """Ask the slave to download source files for a private build.
730
731 :param logger: A logger used for providing debug information.
732+ :return: A list of Deferreds, each of which represents a request
733+ to cache a file.
734 """
735 # The URL to the file in the archive consists of these parts:
736 # archive_url / makePoolPath() / filename
737@@ -164,6 +178,7 @@
738 archive = self.build.archive
739 archive_url = archive.archive_url
740 component_name = self.build.current_component.name
741+ dl = []
742 for source_file in self.build.source_package_release.files:
743 file_name = source_file.libraryfile.filename
744 sha1 = source_file.libraryfile.content.sha1
745@@ -174,8 +189,10 @@
746 logger.debug("Asking builder on %s to ensure it has file %s "
747 "(%s, %s)" % (
748 self._builder.url, file_name, url, sha1))
749- self._builder.slave.sendFileToSlave(
750- sha1, url, "buildd", archive.buildd_secret)
751+ dl.append(
752+ self._builder.slave.sendFileToSlave(
753+ sha1, url, "buildd", archive.buildd_secret))
754+ return dl
755
756 def _extraBuildArgs(self, build):
757 """
758
759=== modified file 'lib/lp/soyuz/scripts/buildd.py'
760--- lib/lp/soyuz/scripts/buildd.py 2010-08-27 11:19:54 +0000
761+++ lib/lp/soyuz/scripts/buildd.py 2010-09-21 09:54:50 +0000
762@@ -276,6 +276,8 @@
763 self.txn.commit()
764
765
766+# XXX: This is the old slave scanner. Julian says it's not running on
767+# production. We should either delete it or update it to use the async apis.
768 class SlaveScanner(LaunchpadCronScript):
769
770 def main(self):
771
772=== modified file 'lib/lp/soyuz/tests/soyuzbuilddhelpers.py'
773--- lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-20 11:52:51 +0000
774+++ lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-21 09:54:50 +0000
775@@ -31,6 +31,8 @@
776 )
777 from lp.testing.sampledata import I386_ARCHITECTURE_NAME
778
779+# XXX: Almost everything in this module will need to be revisited & perhaps
780+# deleted now that we have new APIs for BuilderSlave.
781
782 class MockBuilder:
783 """Emulates a IBuilder class."""
784
785=== modified file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py'
786--- lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-08-20 20:31:18 +0000
787+++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-09-21 09:54:50 +0000
788@@ -41,16 +41,18 @@
789 """See `IBuildFarmJobBehavior`."""
790 chroot = self._getChroot()
791 chroot_sha1 = chroot.content.sha1
792- self._builder.slave.cacheFile(logger, chroot)
793- cookie = self.buildfarmjob.generateSlaveBuildCookie()
794-
795- args = {'arch_tag': self._getDistroArchSeries().architecturetag}
796- args.update(self.buildfarmjob.metadata)
797-
798- filemap = {}
799-
800- self._builder.slave.build(
801- cookie, self.build_type, chroot_sha1, filemap, args)
802+ d = self._builder.slave.cacheFile(logger, chroot)
803+ def got_cache_file(ignored):
804+ cookie = self.buildfarmjob.generateSlaveBuildCookie()
805+
806+ args = {'arch_tag': self._getDistroArchSeries().architecturetag}
807+ args.update(self.buildfarmjob.metadata)
808+
809+ filemap = {}
810+
811+ return self._builder.slave.build(
812+ cookie, self.build_type, chroot_sha1, filemap, args)
813+ return d.addCallback(got_cache_file)
814
815 def _getChroot(self):
816 return self._getDistroArchSeries().getChroot()