Launchpad itself

Merge lp:~jml/launchpad/buildd-deferred-fo-sho into lp:launchpad

  1. buildd-deferred-fo-sho
  2. Merge into devel
Proposed by Jonathan Lange
Status: Merged
Merged at revision: 11801
Proposed branch: lp:~jml/launchpad/buildd-deferred-fo-sho
Merge into: lp:launchpad
Prerequisite: lp:~jml/launchpad/buildd-slavescanner-bustage
Diff against target: 770 lines (+307/-160)
7 files modified
lib/lp/buildmaster/manager.py (+25/-19)
lib/lp/buildmaster/model/builder.py (+38/-17)
lib/lp/buildmaster/tests/test_builder.py (+138/-43)
lib/lp/code/model/recipebuilder.py (+30/-27)
lib/lp/soyuz/model/binarypackagebuildbehavior.py (+56/-40)
lib/lp/soyuz/tests/soyuzbuilddhelpers.py (+8/-4)
lib/lp/translations/model/translationtemplatesbuildbehavior.py (+12/-10)
To merge this branch: bzr merge lp:~jml/launchpad/buildd-deferred-fo-sho
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+36188@code.launchpad.net

Description of the change

Use Deferreds.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/buildmaster/manager.py'
--- lib/lp/buildmaster/manager.py 2010-09-20 10:21:32 +0000
+++ lib/lp/buildmaster/manager.py 2010-09-22 11:33:47 +0000
@@ -283,15 +283,15 @@
283 """Scan the builder and dispatch to it or deal with failures."""283 """Scan the builder and dispatch to it or deal with failures."""
284 self.logger.debug("Scanning builder: %s" % self.builder_name)284 self.logger.debug("Scanning builder: %s" % self.builder_name)
285285
286 try:286 d = self.scan()
287 slave = self.scan()287
288 def got_slave(slave):
288 if slave is None:289 if slave is None:
289 self.scheduleNextScanCycle()290 return self.scheduleNextScanCycle()
290 else:291 else:
291 # XXX: Ought to return Deferred.292 return self.resumeAndDispatch(slave)
292 self.resumeAndDispatch(slave)293
293 except:294 def disaster(error):
294 error = Failure()
295 self.logger.info("Scanning failed with: %s\n%s" %295 self.logger.info("Scanning failed with: %s\n%s" %
296 (error.getErrorMessage(), error.getTraceback()))296 (error.getErrorMessage(), error.getTraceback()))
297297
@@ -307,7 +307,11 @@
307 assessFailureCounts(builder, error.getErrorMessage())307 assessFailureCounts(builder, error.getErrorMessage())
308 transaction.commit()308 transaction.commit()
309309
310 self.scheduleNextScanCycle()310 return self.scheduleNextScanCycle()
311
312 d.addCallback(got_slave)
313 d.addErrback(disaster)
314 return d
311315
312 @write_transaction316 @write_transaction
313 def scan(self):317 def scan(self):
@@ -346,7 +350,7 @@
346 if self.builder.manual:350 if self.builder.manual:
347 self.logger.debug(351 self.logger.debug(
348 '%s is in manual mode, not dispatching.' % self.builder.name)352 '%s is in manual mode, not dispatching.' % self.builder.name)
349 return None353 return defer.succeed(None)
350354
351 # If the builder is marked unavailable, don't dispatch anything.355 # If the builder is marked unavailable, don't dispatch anything.
352 # Additionaly, because builders can be removed from the pool at356 # Additionaly, because builders can be removed from the pool at
@@ -362,7 +366,7 @@
362 "job" % self.builder.name)366 "job" % self.builder.name)
363 job.reset()367 job.reset()
364 transaction.commit()368 transaction.commit()
365 return None369 return defer.succeed(None)
366370
367 # See if there is a job we can dispatch to the builder slave.371 # See if there is a job we can dispatch to the builder slave.
368372
@@ -374,15 +378,17 @@
374 self.builder.name, self.builder.url, self.builder.vm_host)378 self.builder.name, self.builder.url, self.builder.vm_host)
375 # XXX: Passing buildd_slave=slave overwrites the 'slave' property of379 # XXX: Passing buildd_slave=slave overwrites the 'slave' property of
376 # self.builder. Not sure why this is needed yet.380 # self.builder. Not sure why this is needed yet.
377 self.builder.findAndStartJob(buildd_slave=slave)381 d = self.builder.findAndStartJob(buildd_slave=slave)
378 if self.builder.currentjob is not None:382 def job_started(candidate):
379 # After a successful dispatch we can reset the383 if self.builder.currentjob is not None:
380 # failure_count.384 # After a successful dispatch we can reset the
381 self.builder.resetFailureCount()385 # failure_count.
382 transaction.commit()386 self.builder.resetFailureCount()
383 return slave387 transaction.commit()
384388 return slave
385 return None389 else:
390 return None
391 return d.addCallback(job_started)
386392
387 def resumeAndDispatch(self, slave):393 def resumeAndDispatch(self, slave):
388 """Chain the resume and dispatching Deferreds."""394 """Chain the resume and dispatching Deferreds."""
389395
=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py 2010-09-22 11:33:46 +0000
+++ lib/lp/buildmaster/model/builder.py 2010-09-22 11:33:47 +0000
@@ -34,6 +34,7 @@
34 Count,34 Count,
35 Sum,35 Sum,
36 )36 )
37from twisted.internet import defer
37from zope.component import getUtility38from zope.component import getUtility
38from zope.interface import implements39from zope.interface import implements
3940
@@ -166,8 +167,10 @@
166 return self._server.status()167 return self._server.status()
167168
168 def ensurepresent(self, sha1sum, url, username, password):169 def ensurepresent(self, sha1sum, url, username, password):
170 # XXX: Nothing external calls this. Make it private.
169 """Attempt to ensure the given file is present."""171 """Attempt to ensure the given file is present."""
170 return self._server.ensurepresent(sha1sum, url, username, password)172 return defer.succeed(
173 self._server.ensurepresent(sha1sum, url, username, password))
171174
172 def getFile(self, sha_sum):175 def getFile(self, sha_sum):
173 """Construct a file-like object to return the named file."""176 """Construct a file-like object to return the named file."""
@@ -206,13 +209,15 @@
206 logger.debug("Asking builder on %s to ensure it has file %s "209 logger.debug("Asking builder on %s to ensure it has file %s "
207 "(%s, %s)" % (self.urlbase, libraryfilealias.filename,210 "(%s, %s)" % (self.urlbase, libraryfilealias.filename,
208 url, libraryfilealias.content.sha1))211 url, libraryfilealias.content.sha1))
209 self.sendFileToSlave(libraryfilealias.content.sha1, url)212 return self.sendFileToSlave(libraryfilealias.content.sha1, url)
210213
211 def sendFileToSlave(self, sha1, url, username="", password=""):214 def sendFileToSlave(self, sha1, url, username="", password=""):
212 """Helper to send the file at 'url' with 'sha1' to this builder."""215 """Helper to send the file at 'url' with 'sha1' to this builder."""
213 present, info = self.ensurepresent(sha1, url, username, password)216 d = self.ensurepresent(sha1, url, username, password)
214 if not present:217 def check_present((present, info)):
215 raise CannotFetchFile(url, info)218 if not present:
219 raise CannotFetchFile(url, info)
220 return d.addCallback(check_present)
216221
217 def build(self, buildid, builder_type, chroot_sha1, filemap, args):222 def build(self, buildid, builder_type, chroot_sha1, filemap, args):
218 """Build a thing on this build slave.223 """Build a thing on this build slave.
@@ -469,14 +474,19 @@
469474
470 # Do it.475 # Do it.
471 build_queue_item.markAsBuilding(self)476 build_queue_item.markAsBuilding(self)
472 try:477
473 self.current_build_behavior.dispatchBuildToSlave(478 d = self.current_build_behavior.dispatchBuildToSlave(
474 build_queue_item.id, logger)479 build_queue_item.id, logger)
475 except BuildSlaveFailure, e:480
476 logger.debug("Disabling builder: %s" % self.url, exc_info=1)481 def eb_slave_failure(failure):
482 failure.trap(BuildSlaveFailure)
483 e = failure.value
477 self.failBuilder(484 self.failBuilder(
478 "Exception (%s) when setting up to new job" % (e,))485 "Exception (%s) when setting up to new job" % (e,))
479 except CannotFetchFile, e:486
487 def eb_cannot_fetch_file(failure):
488 failure.trap(CannotFetchFile)
489 e = failure.value
480 message = """Slave '%s' (%s) was unable to fetch file.490 message = """Slave '%s' (%s) was unable to fetch file.
481 ****** URL ********491 ****** URL ********
482 %s492 %s
@@ -485,11 +495,19 @@
485 *******************495 *******************
486 """ % (self.name, self.url, e.file_url, e.error_information)496 """ % (self.name, self.url, e.file_url, e.error_information)
487 raise BuildDaemonError(message)497 raise BuildDaemonError(message)
488 except socket.error, e:498
499 def eb_socket_error(failure):
500 failure.trap(socket.error)
501 e = failure.value
489 error_message = "Exception (%s) when setting up new job" % (e,)502 error_message = "Exception (%s) when setting up new job" % (e,)
490 self.handleTimeout(logger, error_message)503 self.handleTimeout(logger, error_message)
491 raise BuildSlaveFailure504 raise BuildSlaveFailure
492505
506 d.addErrback(eb_slave_failure)
507 d.addErrback(eb_cannot_fetch_file)
508 d.addErrback(eb_socket_error)
509 return d
510
493 def failBuilder(self, reason):511 def failBuilder(self, reason):
494 """See IBuilder"""512 """See IBuilder"""
495 # XXX cprov 2007-04-17: ideally we should be able to notify the513 # XXX cprov 2007-04-17: ideally we should be able to notify the
@@ -682,10 +700,13 @@
682 :param candidate: The job to dispatch.700 :param candidate: The job to dispatch.
683 """701 """
684 logger = self._getSlaveScannerLogger()702 logger = self._getSlaveScannerLogger()
685 try:703 d = self.startBuild(candidate, logger)
686 self.startBuild(candidate, logger)704 def warn_on_error(failure):
687 except (BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch), err:705 failure.trap(
706 BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch)
707 err = failure.value
688 logger.warn('Could not build: %s' % err)708 logger.warn('Could not build: %s' % err)
709 return d.addErrback(warn_on_error)
689710
690 def handleTimeout(self, logger, error_message):711 def handleTimeout(self, logger, error_message):
691 """See IBuilder."""712 """See IBuilder."""
@@ -726,8 +747,8 @@
726 if buildd_slave is not None:747 if buildd_slave is not None:
727 self.setSlaveForTesting(buildd_slave)748 self.setSlaveForTesting(buildd_slave)
728749
729 self._dispatchBuildCandidate(candidate)750 d = self._dispatchBuildCandidate(candidate)
730 return candidate751 return d.addCallback(lambda ignored: candidate)
731752
732 def getBuildQueue(self):753 def getBuildQueue(self):
733 """See `IBuilder`."""754 """See `IBuilder`."""
734755
=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
--- lib/lp/buildmaster/tests/test_builder.py 2010-09-21 16:23:35 +0000
+++ lib/lp/buildmaster/tests/test_builder.py 2010-09-22 11:33:47 +0000
@@ -8,8 +8,10 @@
8import socket8import socket
9import xmlrpclib9import xmlrpclib
1010
11from testtools.content import Content11import fixtures
12from testtools.content_type import UTF8_TEXT12
13from twisted.trial.unittest import TestCase as TrialTestCase
14from twisted.web.client import getPage
1315
14from zope.component import getUtility16from zope.component import getUtility
15from zope.security.proxy import removeSecurityProxy17from zope.security.proxy import removeSecurityProxy
@@ -24,10 +26,16 @@
24 )26 )
25from canonical.testing.layers import (27from canonical.testing.layers import (
26 DatabaseFunctionalLayer,28 DatabaseFunctionalLayer,
27 LaunchpadZopelessLayer29 LaunchpadZopelessLayer,
30 TwistedLaunchpadZopelessLayer,
31 TwistedLayer,
28 )32 )
29from lp.buildmaster.enums import BuildStatus33from lp.buildmaster.enums import BuildStatus
30from lp.buildmaster.interfaces.builder import IBuilder, IBuilderSet34from lp.buildmaster.interfaces.builder import (
35 CannotFetchFile,
36 IBuilder,
37 IBuilderSet,
38 )
31from lp.buildmaster.interfaces.buildfarmjobbehavior import (39from lp.buildmaster.interfaces.buildfarmjobbehavior import (
32 IBuildFarmJobBehavior,40 IBuildFarmJobBehavior,
33 )41 )
@@ -49,9 +57,12 @@
49 )57 )
50from lp.soyuz.tests.test_publishing import SoyuzTestPublisher58from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
51from lp.testing import (59from lp.testing import (
52 TestCase,60 ANONYMOUS,
61 login_as,
62 logout,
53 TestCaseWithFactory,63 TestCaseWithFactory,
54 )64 )
65from lp.testing.factory import LaunchpadObjectFactory
55from lp.testing.fakemethod import FakeMethod66from lp.testing.fakemethod import FakeMethod
5667
5768
@@ -467,19 +478,11 @@
467 self.builder.current_build_behavior, BinaryPackageBuildBehavior)478 self.builder.current_build_behavior, BinaryPackageBuildBehavior)
468479
469480
470class TestSlave(TestCase):481class SlaveTestHelpers(fixtures.Fixture):
471 """
472 Integration tests for BuilderSlave that verify how it works against a
473 real slave server.
474 """
475
476 # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
477 # BuilderSlave in buildd-slave.txt and in other places. The tests here
478 # ought to become the canonical tests for BuilderSlave vs running buildd
479 # XML-RPC server interaction.
480482
481 # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`.483 # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`.
482 TEST_URL = 'http://localhost:8221/rpc/'484 BASE_URL = 'http://localhost:8221'
485 TEST_URL = '%s/rpc/' % (BASE_URL,)
483486
484 def getServerSlave(self):487 def getServerSlave(self):
485 """Set up a test build slave server.488 """Set up a test build slave server.
@@ -488,11 +491,14 @@
488 """491 """
489 tachandler = BuilddSlaveTestSetup()492 tachandler = BuilddSlaveTestSetup()
490 tachandler.setUp()493 tachandler.setUp()
491 def addLogFile(exc_info):494 # Basically impossible to do this w/ TrialTestCase. But it would be
492 self.addDetail(495 # really nice to keep it.
493 'xmlrpc-log-file',496 #
494 Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))497 # def addLogFile(exc_info):
495 self.addOnException(addLogFile)498 # self.addDetail(
499 # 'xmlrpc-log-file',
500 # Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read()))
501 # self.addOnException(addLogFile)
496 self.addCleanup(tachandler.tearDown)502 self.addCleanup(tachandler.tearDown)
497 return tachandler503 return tachandler
498504
@@ -527,7 +533,7 @@
527 :return: The build id returned by the slave.533 :return: The build id returned by the slave.
528 """534 """
529 if build_id is None:535 if build_id is None:
530 build_id = self.getUniqueString()536 build_id = 'random-build-id'
531 tachandler = self.getServerSlave()537 tachandler = self.getServerSlave()
532 chroot_file = 'fake-chroot'538 chroot_file = 'fake-chroot'
533 dsc_file = 'thing'539 dsc_file = 'thing'
@@ -537,10 +543,30 @@
537 build_id, 'debian', chroot_file, {'.dsc': dsc_file},543 build_id, 'debian', chroot_file, {'.dsc': dsc_file},
538 {'ogrecomponent': 'main'})544 {'ogrecomponent': 'main'})
539545
546
547class TestSlave(TrialTestCase):
548 """
549 Integration tests for BuilderSlave that verify how it works against a
550 real slave server.
551 """
552
553 layer = TwistedLayer
554
555 def setUp(self):
556 super(TestSlave, self).setUp()
557 self.slave_helper = SlaveTestHelpers()
558 self.slave_helper.setUp()
559 self.addCleanup(self.slave_helper.cleanUp)
560
561 # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for
562 # BuilderSlave in buildd-slave.txt and in other places. The tests here
563 # ought to become the canonical tests for BuilderSlave vs running buildd
564 # XML-RPC server interaction.
565
540 def test_abort(self):566 def test_abort(self):
541 slave = self.getClientSlave()567 slave = self.slave_helper.getClientSlave()
542 # We need to be in a BUILDING state before we can abort.568 # We need to be in a BUILDING state before we can abort.
543 self.triggerGoodBuild(slave)569 self.slave_helper.triggerGoodBuild(slave)
544 result = slave.abort()570 result = slave.abort()
545 self.assertEqual(result, BuilderStatus.ABORTING)571 self.assertEqual(result, BuilderStatus.ABORTING)
546572
@@ -549,8 +575,8 @@
549 # valid chroot & filemaps works and returns a BuilderStatus of575 # valid chroot & filemaps works and returns a BuilderStatus of
550 # BUILDING.576 # BUILDING.
551 build_id = 'some-id'577 build_id = 'some-id'
552 slave = self.getClientSlave()578 slave = self.slave_helper.getClientSlave()
553 result = self.triggerGoodBuild(slave, build_id)579 result = self.slave_helper.triggerGoodBuild(slave, build_id)
554 self.assertEqual([BuilderStatus.BUILDING, build_id], result)580 self.assertEqual([BuilderStatus.BUILDING, build_id], result)
555581
556 def test_clean(self):582 def test_clean(self):
@@ -564,15 +590,15 @@
564 def test_echo(self):590 def test_echo(self):
565 # Calling 'echo' contacts the server which returns the arguments we591 # Calling 'echo' contacts the server which returns the arguments we
566 # gave it.592 # gave it.
567 self.getServerSlave()593 self.slave_helper.getServerSlave()
568 slave = self.getClientSlave()594 slave = self.slave_helper.getClientSlave()
569 result = slave.echo('foo', 'bar', 42)595 result = slave.echo('foo', 'bar', 42)
570 self.assertEqual(['foo', 'bar', 42], result)596 self.assertEqual(['foo', 'bar', 42], result)
571597
572 def test_info(self):598 def test_info(self):
573 # Calling 'info' gets some information about the slave.599 # Calling 'info' gets some information about the slave.
574 self.getServerSlave()600 self.slave_helper.getServerSlave()
575 slave = self.getClientSlave()601 slave = self.slave_helper.getClientSlave()
576 result = slave.info()602 result = slave.info()
577 # We're testing the hard-coded values, since the version is hard-coded603 # We're testing the hard-coded values, since the version is hard-coded
578 # into the remote slave, the supported build managers are hard-coded604 # into the remote slave, the supported build managers are hard-coded
@@ -588,17 +614,17 @@
588 def test_initial_status(self):614 def test_initial_status(self):
589 # Calling 'status' returns the current status of the slave. The615 # Calling 'status' returns the current status of the slave. The
590 # initial status is IDLE.616 # initial status is IDLE.
591 self.getServerSlave()617 self.slave_helper.getServerSlave()
592 slave = self.getClientSlave()618 slave = self.slave_helper.getClientSlave()
593 status = slave.status()619 status = slave.status()
594 self.assertEqual([BuilderStatus.IDLE, ''], status)620 self.assertEqual([BuilderStatus.IDLE, ''], status)
595621
596 def test_status_after_build(self):622 def test_status_after_build(self):
597 # Calling 'status' returns the current status of the slave. After a623 # Calling 'status' returns the current status of the slave. After a
598 # build has been triggered, the status is BUILDING.624 # build has been triggered, the status is BUILDING.
599 slave = self.getClientSlave()625 slave = self.slave_helper.getClientSlave()
600 build_id = 'status-build-id'626 build_id = 'status-build-id'
601 self.triggerGoodBuild(slave, build_id)627 self.slave_helper.triggerGoodBuild(slave, build_id)
602 status = slave.status()628 status = slave.status()
603 self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2])629 self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2])
604 [log_file] = status[2:]630 [log_file] = status[2:]
@@ -606,15 +632,84 @@
606632
607 def test_ensurepresent_not_there(self):633 def test_ensurepresent_not_there(self):
608 # ensurepresent checks to see if a file is there.634 # ensurepresent checks to see if a file is there.
609 self.getServerSlave()635 self.slave_helper.getServerSlave()
610 slave = self.getClientSlave()636 slave = self.slave_helper.getClientSlave()
611 result = slave.ensurepresent('blahblah', None, None, None)637 d = slave.ensurepresent('blahblah', None, None, None)
612 self.assertEqual([False, 'No URL'], result)638 d.addCallback(self.assertEqual, [False, 'No URL'])
639 return d
613640
614 def test_ensurepresent_actually_there(self):641 def test_ensurepresent_actually_there(self):
615 # ensurepresent checks to see if a file is there.642 # ensurepresent checks to see if a file is there.
616 tachandler = self.getServerSlave()643 tachandler = self.slave_helper.getServerSlave()
617 slave = self.getClientSlave()644 slave = self.slave_helper.getClientSlave()
618 self.makeCacheFile(tachandler, 'blahblah')645 self.slave_helper.makeCacheFile(tachandler, 'blahblah')
619 result = slave.ensurepresent('blahblah', None, None, None)646 d = slave.ensurepresent('blahblah', None, None, None)
620 self.assertEqual([True, 'No URL'], result)647 d.addCallback(self.assertEqual, [True, 'No URL'])
648 return d
649
650 def test_sendFileToSlave_not_there(self):
651 self.slave_helper.getServerSlave()
652 slave = self.slave_helper.getClientSlave()
653 d = slave.sendFileToSlave('blahblah', None, None, None)
654 return self.assertFailure(d, CannotFetchFile)
655
656 def test_sendFileToSlave_actually_there(self):
657 tachandler = self.slave_helper.getServerSlave()
658 slave = self.slave_helper.getClientSlave()
659 self.slave_helper.makeCacheFile(tachandler, 'blahblah')
660 d = slave.sendFileToSlave('blahblah', None, None, None)
661 def check_present(ignored):
662 d = slave.ensurepresent('blahblah', None, None, None)
663 return d.addCallback(self.assertEqual, [True, 'No URL'])
664 d.addCallback(check_present)
665 return d
666
667
668class TestSlaveWithLibrarian(TrialTestCase):
669 """Tests that need more of Launchpad to run."""
670
671 layer = TwistedLaunchpadZopelessLayer
672
673 def setUp(self):
674 super(TestSlaveWithLibrarian, self)
675 self.slave_helper = SlaveTestHelpers()
676 self.slave_helper.setUp()
677 self.addCleanup(self.slave_helper.cleanUp)
678 self.factory = LaunchpadObjectFactory()
679 login_as(ANONYMOUS)
680 self.addCleanup(logout)
681
682 def test_ensurepresent_librarian(self):
683 # ensurepresent, when given an http URL for a file will download the
684 # file from that URL and report that the file is present, and it was
685 # downloaded.
686
687 # Use the Librarian because it's a "convenient" web server.
688 lf = self.factory.makeLibraryFileAlias(
689 'HelloWorld.txt', content="Hello World")
690 self.layer.txn.commit()
691 self.slave_helper.getServerSlave()
692 slave = self.slave_helper.getClientSlave()
693 d = slave.ensurepresent(
694 lf.content.sha1, lf.http_url, "", "")
695 d.addCallback(self.assertEqual, [True, 'Download'])
696 return d
697
698 def test_retrieve_files_from_filecache(self):
699 # Files that are present on the slave can be downloaded with a
700 # filename made from the sha1 of the content underneath the
701 # 'filecache' directory.
702 content = "Hello World"
703 lf = self.factory.makeLibraryFileAlias(
704 'HelloWorld.txt', content=content)
705 self.layer.txn.commit()
706 expected_url = '%s/filecache/%s' % (
707 self.slave_helper.BASE_URL, lf.content.sha1)
708 self.slave_helper.getServerSlave()
709 slave = self.slave_helper.getClientSlave()
710 d = slave.ensurepresent(
711 lf.content.sha1, lf.http_url, "", "")
712 def check_file(ignored):
713 d = getPage(expected_url.encode('utf8'))
714 return d.addCallback(self.assertEqual, content)
715 return d.addCallback(check_file)
621716
=== modified file 'lib/lp/code/model/recipebuilder.py'
--- lib/lp/code/model/recipebuilder.py 2010-08-20 20:31:18 +0000
+++ lib/lp/code/model/recipebuilder.py 2010-09-22 11:33:47 +0000
@@ -122,33 +122,36 @@
122 if chroot is None:122 if chroot is None:
123 raise CannotBuild("Unable to find a chroot for %s" %123 raise CannotBuild("Unable to find a chroot for %s" %
124 distroarchseries.displayname)124 distroarchseries.displayname)
125 self._builder.slave.cacheFile(logger, chroot)125 d = self._builder.slave.cacheFile(logger, chroot)
126126
127 # Generate a string which can be used to cross-check when obtaining127 def got_cache_file(ignored):
128 # results so we know we are referring to the right database object in128 # Generate a string which can be used to cross-check when obtaining
129 # subsequent runs.129 # results so we know we are referring to the right database object in
130 buildid = "%s-%s" % (self.build.id, build_queue_id)130 # subsequent runs.
131 cookie = self.buildfarmjob.generateSlaveBuildCookie()131 buildid = "%s-%s" % (self.build.id, build_queue_id)
132 chroot_sha1 = chroot.content.sha1132 cookie = self.buildfarmjob.generateSlaveBuildCookie()
133 logger.debug(133 chroot_sha1 = chroot.content.sha1
134 "Initiating build %s on %s" % (buildid, self._builder.url))134 logger.debug(
135135 "Initiating build %s on %s" % (buildid, self._builder.url))
136 args = self._extraBuildArgs(distroarchseries, logger)136
137 status, info = self._builder.slave.build(137 args = self._extraBuildArgs(distroarchseries, logger)
138 cookie, "sourcepackagerecipe", chroot_sha1, {}, args)138 # XXX: Soon to be async
139 message = """%s (%s):139 status, info = self._builder.slave.build(
140 ***** RESULT *****140 cookie, "sourcepackagerecipe", chroot_sha1, {}, args)
141 %s141 message = """%s (%s):
142 %s: %s142 ***** RESULT *****
143 ******************143 %s
144 """ % (144 %s: %s
145 self._builder.name,145 ******************
146 self._builder.url,146 """ % (
147 args,147 self._builder.name,
148 status,148 self._builder.url,
149 info,149 args,
150 )150 status,
151 logger.info(message)151 info,
152 )
153 logger.info(message)
154 return d.addCallback(got_cache_file)
152155
153 def verifyBuildRequest(self, logger):156 def verifyBuildRequest(self, logger):
154 """Assert some pre-build checks.157 """Assert some pre-build checks.
155158
=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
--- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-22 11:33:46 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-22 11:33:47 +0000
@@ -11,6 +11,7 @@
11 'BinaryPackageBuildBehavior',11 'BinaryPackageBuildBehavior',
12 ]12 ]
1313
14from twisted.internet import defer
14from zope.interface import implements15from zope.interface import implements
1516
16from canonical.launchpad.webapp import urlappend17from canonical.launchpad.webapp import urlappend
@@ -38,56 +39,66 @@
38 logger.info("startBuild(%s, %s, %s, %s)", self._builder.url,39 logger.info("startBuild(%s, %s, %s, %s)", self._builder.url,
39 spr.name, spr.version, self.build.pocket.title)40 spr.name, spr.version, self.build.pocket.title)
4041
41 def dispatchBuildToSlave(self, build_queue_id, logger):42 def _buildFilemapStructure(self, ignored, logger):
42 """See `IBuildFarmJobBehavior`."""
43
44 # Start the binary package build on the slave builder. First
45 # we send the chroot.
46 chroot = self.build.distro_arch_series.getChroot()
47 self._builder.slave.cacheFile(logger, chroot)
48
49 # Build filemap structure with the files required in this build43 # Build filemap structure with the files required in this build
50 # and send them to the slave.44 # and send them to the slave.
51 # If the build is private we tell the slave to get the files from the45 # If the build is private we tell the slave to get the files from the
52 # archive instead of the librarian because the slaves cannot46 # archive instead of the librarian because the slaves cannot
53 # access the restricted librarian.47 # access the restricted librarian.
48 dl = []
54 private = self.build.archive.private49 private = self.build.archive.private
55 if private:50 if private:
56 self._cachePrivateSourceOnSlave(logger)51 dl.extend(self._cachePrivateSourceOnSlave(logger))
57 filemap = {}52 filemap = {}
58 for source_file in self.build.source_package_release.files:53 for source_file in self.build.source_package_release.files:
59 lfa = source_file.libraryfile54 lfa = source_file.libraryfile
60 filemap[lfa.filename] = lfa.content.sha155 filemap[lfa.filename] = lfa.content.sha1
61 if not private:56 if not private:
62 self._builder.slave.cacheFile(logger, source_file.libraryfile)57 dl.append(
6358 self._builder.slave.cacheFile(
64 # Generate a string which can be used to cross-check when obtaining59 logger, source_file.libraryfile))
65 # results so we know we are referring to the right database object in60 d = defer.gatherResults(dl)
66 # subsequent runs.61 return d.addCallback(lambda ignored: filemap)
67 buildid = "%s-%s" % (self.build.id, build_queue_id)62
68 cookie = self.buildfarmjob.generateSlaveBuildCookie()63 def dispatchBuildToSlave(self, build_queue_id, logger):
69 chroot_sha1 = chroot.content.sha164 """See `IBuildFarmJobBehavior`."""
70 logger.debug(65
71 "Initiating build %s on %s" % (buildid, self._builder.url))66 # Start the binary package build on the slave builder. First
7267 # we send the chroot.
73 args = self._extraBuildArgs(self.build)68 chroot = self.build.distro_arch_series.getChroot()
74 status, info = self._builder.slave.build(69 d = self._builder.slave.cacheFile(logger, chroot)
75 cookie, "binarypackage", chroot_sha1, filemap, args)70 d.addCallback(self._buildFilemapStructure, logger)
76 message = """%s (%s):71
77 ***** RESULT *****72 def got_filemap(filemap):
78 %s73 # Generate a string which can be used to cross-check when obtaining
79 %s74 # results so we know we are referring to the right database object in
80 %s: %s75 # subsequent runs.
81 ******************76 buildid = "%s-%s" % (self.build.id, build_queue_id)
82 """ % (77 cookie = self.buildfarmjob.generateSlaveBuildCookie()
83 self._builder.name,78 chroot_sha1 = chroot.content.sha1
84 self._builder.url,79 logger.debug(
85 filemap,80 "Initiating build %s on %s" % (buildid, self._builder.url))
86 args,81
87 status,82 args = self._extraBuildArgs(self.build)
88 info,83 status, info = self._builder.slave.build(
89 )84 cookie, "binarypackage", chroot_sha1, filemap, args)
90 logger.info(message)85 message = """%s (%s):
86 ***** RESULT *****
87 %s
88 %s
89 %s: %s
90 ******************
91 """ % (
92 self._builder.name,
93 self._builder.url,
94 filemap,
95 args,
96 status,
97 info,
98 )
99 logger.info(message)
100
101 return d.addCallback(got_filemap)
91102
92 def verifyBuildRequest(self, logger):103 def verifyBuildRequest(self, logger):
93 """Assert some pre-build checks.104 """Assert some pre-build checks.
@@ -154,6 +165,8 @@
154 """Ask the slave to download source files for a private build.165 """Ask the slave to download source files for a private build.
155166
156 :param logger: A logger used for providing debug information.167 :param logger: A logger used for providing debug information.
168 :return: A list of Deferreds, each of which represents a request
169 to cache a file.
157 """170 """
158 # The URL to the file in the archive consists of these parts:171 # The URL to the file in the archive consists of these parts:
159 # archive_url / makePoolPath() / filename172 # archive_url / makePoolPath() / filename
@@ -165,6 +178,7 @@
165 archive = self.build.archive178 archive = self.build.archive
166 archive_url = archive.archive_url179 archive_url = archive.archive_url
167 component_name = self.build.current_component.name180 component_name = self.build.current_component.name
181 dl = []
168 for source_file in self.build.source_package_release.files:182 for source_file in self.build.source_package_release.files:
169 file_name = source_file.libraryfile.filename183 file_name = source_file.libraryfile.filename
170 sha1 = source_file.libraryfile.content.sha1184 sha1 = source_file.libraryfile.content.sha1
@@ -175,8 +189,10 @@
175 logger.debug("Asking builder on %s to ensure it has file %s "189 logger.debug("Asking builder on %s to ensure it has file %s "
176 "(%s, %s)" % (190 "(%s, %s)" % (
177 self._builder.url, file_name, url, sha1))191 self._builder.url, file_name, url, sha1))
178 self._builder.slave.sendFileToSlave(192 dl.append(
179 sha1, url, "buildd", archive.buildd_secret)193 self._builder.slave.sendFileToSlave(
194 sha1, url, "buildd", archive.buildd_secret))
195 return dl
180196
181 def _extraBuildArgs(self, build):197 def _extraBuildArgs(self, build):
182 """198 """
183199
=== modified file 'lib/lp/soyuz/tests/soyuzbuilddhelpers.py'
--- lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-22 11:33:46 +0000
+++ lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-22 11:33:47 +0000
@@ -19,6 +19,8 @@
19from StringIO import StringIO19from StringIO import StringIO
20import xmlrpclib20import xmlrpclib
2121
22from twisted.internet import defer
23
22from lp.buildmaster.interfaces.builder import CannotFetchFile24from lp.buildmaster.interfaces.builder import CannotFetchFile
23from lp.buildmaster.model.builder import (25from lp.buildmaster.model.builder import (
24 rescueBuilderIfLost,26 rescueBuilderIfLost,
@@ -99,7 +101,7 @@
99101
100 def ensurepresent(self, sha1, url, user=None, password=None):102 def ensurepresent(self, sha1, url, user=None, password=None):
101 self.call_log.append(('ensurepresent', url, user, password))103 self.call_log.append(('ensurepresent', url, user, password))
102 return True, None104 return defer.succeed((True, None))
103105
104 def build(self, buildid, buildtype, chroot, filemap, args):106 def build(self, buildid, buildtype, chroot, filemap, args):
105 self.call_log.append(107 self.call_log.append(
@@ -123,9 +125,11 @@
123125
124 def sendFileToSlave(self, sha1, url, username="", password=""):126 def sendFileToSlave(self, sha1, url, username="", password=""):
125 self.call_log.append('sendFileToSlave')127 self.call_log.append('sendFileToSlave')
126 present, info = self.ensurepresent(sha1, url, username, password)128 d = self.ensurepresent(sha1, url, username, password)
127 if not present:129 def check_present((present, info)):
128 raise CannotFetchFile(url, info)130 if not present:
131 raise CannotFetchFile(url, info)
132 return d.addCallback(check_present)
129133
130 def cacheFile(self, logger, libraryfilealias):134 def cacheFile(self, logger, libraryfilealias):
131 self.call_log.append('cacheFile')135 self.call_log.append('cacheFile')
132136
=== modified file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py'
--- lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-08-20 20:31:18 +0000
+++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-09-22 11:33:47 +0000
@@ -41,16 +41,18 @@
41 """See `IBuildFarmJobBehavior`."""41 """See `IBuildFarmJobBehavior`."""
42 chroot = self._getChroot()42 chroot = self._getChroot()
43 chroot_sha1 = chroot.content.sha143 chroot_sha1 = chroot.content.sha1
44 self._builder.slave.cacheFile(logger, chroot)44 d = self._builder.slave.cacheFile(logger, chroot)
45 cookie = self.buildfarmjob.generateSlaveBuildCookie()45 def got_cache_file(ignored):
4646 cookie = self.buildfarmjob.generateSlaveBuildCookie()
47 args = {'arch_tag': self._getDistroArchSeries().architecturetag}47
48 args.update(self.buildfarmjob.metadata)48 args = {'arch_tag': self._getDistroArchSeries().architecturetag}
4949 args.update(self.buildfarmjob.metadata)
50 filemap = {}50
5151 filemap = {}
52 self._builder.slave.build(52
53 cookie, self.build_type, chroot_sha1, filemap, args)53 return self._builder.slave.build(
54 cookie, self.build_type, chroot_sha1, filemap, args)
55 return d.addCallback(got_cache_file)
5456
55 def _getChroot(self):57 def _getChroot(self):
56 return self._getDistroArchSeries().getChroot()58 return self._getDistroArchSeries().getChroot()