Launchpad itself

Merge lp:~rockstar/launchpad/branch-scanner-job into lp:launchpad/db-devel

  1. branch-scanner-job
  2. Merge into db-devel
Proposed by Paul Hummer
Status: Merged
Merged at revision: not available
Proposed branch: lp:~rockstar/launchpad/branch-scanner-job
Merge into: lp:launchpad/db-devel
Prerequisite: lp:~rockstar/launchpad/fix-create-branch-and-tree
Diff against target: 1423 lines (+630/-145)
21 files modified
lib/lp/buildmaster/interfaces/buildfarmjob.py (+6/-6)
lib/lp/buildmaster/manager.py (+2/-10)
lib/lp/code/interfaces/branchjob.py (+21/-0)
lib/lp/code/model/branchjob.py (+37/-4)
lib/lp/code/model/tests/test_branchjob.py (+53/-3)
lib/lp/code/templates/branchmergeproposal-macros.pt (+1/-1)
lib/lp/soyuz/doc/buildd-dispatching.txt (+17/-11)
lib/lp/soyuz/doc/buildd-slavescanner.txt (+31/-30)
lib/lp/soyuz/interfaces/builder.py (+8/-18)
lib/lp/soyuz/interfaces/publishing.py (+16/-15)
lib/lp/soyuz/model/builder.py (+41/-9)
lib/lp/soyuz/model/publishing.py (+2/-6)
lib/lp/soyuz/scripts/buildd.py (+2/-4)
lib/lp/soyuz/stories/webservice/xx-source-package-publishing.txt (+14/-16)
lib/lp/soyuz/tests/test_builder.py (+16/-11)
lib/lp/translations/configure.zcml (+29/-0)
lib/lp/translations/interfaces/translationtemplatesbuildjob.py (+37/-0)
lib/lp/translations/model/translationtemplatesbuildbehavior.py (+79/-0)
lib/lp/translations/model/translationtemplatesbuildjob.py (+85/-0)
lib/lp/translations/tests/test_translationtemplatesbuildjob.py (+132/-0)
utilities/sourcedeps.conf (+1/-1)
To merge this branch: bzr merge lp:~rockstar/launchpad/branch-scanner-job
Reviewer Review Type Date Requested Status
Muharem Hrnjadovic (community) Approve
Review via email: mp+17132@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Paul Hummer (rockstar) wrote :

This branch creates a BranchScanJob and accompanying tests. The end goal of
all this work is so that the branch scanner can be converted into a job. The
next step is to create a script that will run the branch scan job, and then
have the mirror-puller create scan jobs after its done mirroring things.

Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

Hello Paul, this looks good. Please do make the minor fixes ("make lint" fixes etc.) we discussed prior to merging the branch.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py'
2--- lib/lp/buildmaster/interfaces/buildfarmjob.py 2009-12-24 14:18:35 +0000
3+++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-13 03:21:24 +0000
4@@ -46,24 +46,24 @@
5 Build a package from a bazaar branch and a recipe.
6 """)
7
8- TRANSLATION = DBItem(4, """
9- TranslationJob
10+ TRANSLATIONTEMPLATESBUILD = DBItem(4, """
11+ TranslationTemplatesBuildJob
12
13- Perform a translation job.
14+ Generate translation templates from a bazaar branch.
15 """)
16
17
18 class IBuildFarmJob(Interface):
19- """Operations that Soyuz build farm jobs must implement."""
20+ """Operations that jobs for the build farm must implement."""
21
22 def score():
23 """Calculate a job score appropriate for the job type in question."""
24
25 def getLogFileName():
26- """The preferred file name for the log of this Soyuz job."""
27+ """The preferred file name for this job's log."""
28
29 def getName():
30- """An appropriate name for this Soyuz job."""
31+ """An appropriate name for this job."""
32
33 def jobStarted():
34 """'Job started' life cycle event, handle as appropriate."""
35
36=== modified file 'lib/lp/buildmaster/manager.py'
37--- lib/lp/buildmaster/manager.py 2009-07-26 14:19:49 +0000
38+++ lib/lp/buildmaster/manager.py 2010-01-13 03:21:24 +0000
39@@ -340,19 +340,11 @@
40 transaction.commit()
41 continue
42
43- candidate = builder.findBuildCandidate()
44- if candidate is None:
45- self.logger.debug(
46- "No build candidates available for builder.")
47- continue
48-
49 slave = RecordingSlave(builder.name, builder.url, builder.vm_host)
50- builder.setSlaveForTesting(slave)
51-
52- builder.dispatchBuildCandidate(candidate)
53+ candidate = builder.findAndStartJob(buildd_slave=slave)
54 if builder.currentjob is not None:
55 recording_slaves.append(slave)
56- transaction.commit()
57+ transaction.commit()
58
59 return recording_slaves
60
61
62=== modified file 'lib/lp/code/interfaces/branchjob.py'
63--- lib/lp/code/interfaces/branchjob.py 2010-01-07 04:58:38 +0000
64+++ lib/lp/code/interfaces/branchjob.py 2010-01-13 03:21:24 +0000
65@@ -13,6 +13,8 @@
66 'IBranchJob',
67 'IBranchDiffJob',
68 'IBranchDiffJobSource',
69+ 'IBranchScanJob',
70+ 'IBranchScanJobSource',
71 'IBranchUpgradeJob',
72 'IBranchUpgradeJobSource',
73 'IReclaimBranchSpaceJob',
74@@ -74,6 +76,25 @@
75 """
76
77
78+class IBranchScanJob(Interface):
79+ """ A job to scan branches."""
80+
81+ def run():
82+ """Scan a branch for new revisions."""
83+
84+
85+class IBranchScanJobSource(Interface):
86+
87+ def create(branch):
88+ """Scan a branch for new revisions.
89+
90+ :param branch: The database branch to upgrade.
91+ """
92+
93+ def iterReady():
94+ """Iterate through all IBranchScanJobs."""
95+
96+
97 class IBranchUpgradeJob(IRunnableJob):
98 """A job to upgrade branches with out-of-date formats."""
99
100
101=== modified file 'lib/lp/code/model/branchjob.py'
102--- lib/lp/code/model/branchjob.py 2010-01-07 04:57:13 +0000
103+++ lib/lp/code/model/branchjob.py 2010-01-13 03:21:24 +0000
104@@ -48,6 +48,7 @@
105 from lp.code.model.branchmergeproposal import BranchMergeProposal
106 from lp.code.model.diff import StaticDiff
107 from lp.code.model.revision import RevisionSet
108+from lp.codehosting.scanner.bzrsync import BzrSync
109 from lp.codehosting.vfs import branch_id_to_path, get_multi_server
110 from lp.services.job.model.job import Job
111 from lp.services.job.interfaces.job import JobStatus
112@@ -59,10 +60,11 @@
113 BranchMergeProposalStatus, BranchSubscriptionDiffSize,
114 BranchSubscriptionNotificationLevel)
115 from lp.code.interfaces.branchjob import (
116- IBranchDiffJob, IBranchDiffJobSource, IBranchJob, IBranchUpgradeJob,
117- IBranchUpgradeJobSource, IReclaimBranchSpaceJob,
118- IReclaimBranchSpaceJobSource, IRevisionsAddedJob, IRevisionMailJob,
119- IRevisionMailJobSource, IRosettaUploadJob, IRosettaUploadJobSource)
120+ IBranchDiffJob, IBranchDiffJobSource, IBranchJob, IBranchScanJob,
121+ IBranchScanJobSource, IBranchUpgradeJob, IBranchUpgradeJobSource,
122+ IReclaimBranchSpaceJob, IReclaimBranchSpaceJobSource, IRevisionsAddedJob,
123+ IRevisionMailJob, IRevisionMailJobSource, IRosettaUploadJob,
124+ IRosettaUploadJobSource)
125 from lp.translations.interfaces.translations import (
126 TranslationsBranchImportMode)
127 from lp.translations.interfaces.translationimportqueue import (
128@@ -119,6 +121,17 @@
129 from disk.
130 """)
131
132+ TRANSLATION_TEMPLATES_BUILD = DBItem(6, """
133+ Generate translation templates
134+
135+ This job generates translations templates from a source branch.
136+ """)
137+
138+ SCAN_BRANCH = DBItem(7, """
139+
140+ Scan Branch
141+ This job scans a branch for new revisions.
142+ """)
143
144 class BranchJob(SQLBase):
145 """Base class for jobs related to branches."""
146@@ -243,6 +256,26 @@
147 return static_diff
148
149
150+class BranchScanJob(BranchJobDerived):
151+ """A Job that scans a branch for new revisions."""
152+
153+ implements(IBranchScanJob)
154+
155+ classProvides(IBranchScanJobSource)
156+ class_job_type = BranchJobType.SCAN_BRANCH
157+
158+ @classmethod
159+ def create(cls, branch):
160+ """See `IBranchUpgradeJobSource`."""
161+ branch_job = BranchJob(branch, BranchJobType.SCAN_BRANCH, {})
162+ return cls(branch_job)
163+
164+ def run(self):
165+ """See `IBranchScanJob`."""
166+ bzrsync = BzrSync(self.branch)
167+ bzrsync.syncBranchAndClose()
168+
169+
170 class BranchUpgradeJob(BranchJobDerived):
171 """A Job that upgrades branches to the current stable format."""
172
173
174=== modified file 'lib/lp/code/model/tests/test_branchjob.py'
175--- lib/lp/code/model/tests/test_branchjob.py 2010-01-11 05:08:06 +0000
176+++ lib/lp/code/model/tests/test_branchjob.py 2010-01-13 03:21:24 +0000
177@@ -46,11 +46,12 @@
178 BranchSubscriptionDiffSize, BranchSubscriptionNotificationLevel,
179 CodeReviewNotificationLevel)
180 from lp.code.interfaces.branchjob import (
181- IBranchDiffJob, IBranchJob, IBranchUpgradeJob, IReclaimBranchSpaceJob,
182- IReclaimBranchSpaceJobSource, IRevisionMailJob, IRosettaUploadJob)
183+ IBranchDiffJob, IBranchJob, IBranchScanJob, IBranchUpgradeJob,
184+ IReclaimBranchSpaceJob, IReclaimBranchSpaceJobSource, IRevisionMailJob,
185+ IRosettaUploadJob)
186 from lp.code.model.branchjob import (
187 BranchDiffJob, BranchJob, BranchJobDerived, BranchJobType,
188- BranchUpgradeJob, ReclaimBranchSpaceJob, RevisionMailJob,
189+ BranchScanJob, BranchUpgradeJob, ReclaimBranchSpaceJob, RevisionMailJob,
190 RevisionsAddedJob, RosettaUploadJob)
191 from lp.code.model.branchrevision import BranchRevision
192 from lp.code.model.revision import RevisionSet
193@@ -112,6 +113,7 @@
194 def test_run_diff_content(self):
195 """Ensure that run generates expected diff."""
196 self.useBzrBranches()
197+<<<<<<< TREE
198
199 tree_location = tempfile.mkdtemp()
200 self.addCleanup(lambda: shutil.rmtree(tree_location))
201@@ -119,6 +121,16 @@
202 branch, tree = self.create_branch_and_tree(tree_location=tree_location)
203 tree_file = os.path.join(tree_location, 'file')
204 open(tree_file, 'wb').write('foo\n')
205+=======
206+
207+ tree_location = tempfile.mkdtemp()
208+ self.addCleanup(lambda: shutil.rmtree(tree_location))
209+
210+ branch, tree = self.create_branch_and_tree(
211+ tree_location=tree_location)
212+ tree_file = os.path.join(tree_location, 'file')
213+ open(tree_file, 'wb').write('foo\n')
214+>>>>>>> MERGE-SOURCE
215 tree.add('file')
216 tree.commit('First commit')
217 open(tree_file, 'wb').write('bar\n')
218@@ -184,6 +196,44 @@
219 self.assertIsInstance(diff.diff.text, str)
220
221
222+class TestBranchScanJob(TestCaseWithFactory):
223+ """Tests for `BranchScanJob`."""
224+
225+ layer = LaunchpadZopelessLayer
226+
227+ def test_providesInterface(self):
228+ """Ensure that BranchScanJob implements IBranchScanJob."""
229+ branch = self.factory.makeAnyBranch()
230+ job = BranchScanJob.create(branch)
231+ verifyObject(IBranchScanJob, job)
232+
233+ def test_run(self):
234+ """Ensure the job scans the branch."""
235+ self.useBzrBranches()
236+
237+ db_branch, bzr_tree = self.create_branch_and_tree()
238+ bzr_tree.commit('First commit', rev_id='rev1')
239+ bzr_tree.commit('Second commit', rev_id='rev2')
240+ bzr_tree.commit('Third commit', rev_id='rev3')
241+ LaunchpadZopelessLayer.commit()
242+
243+ job = BranchScanJob.create(db_branch)
244+ LaunchpadZopelessLayer.switchDbUser(config.branchscanner.dbuser)
245+ job.run()
246+ LaunchpadZopelessLayer.switchDbUser(config.launchpad.dbuser)
247+
248+ self.assertEqual(db_branch.revision_count, 3)
249+
250+ bzr_tree.commit('Fourth commit', rev_id='rev4')
251+ bzr_tree.commit('Fifth commit', rev_id='rev5')
252+
253+ job = BranchScanJob.create(db_branch)
254+ LaunchpadZopelessLayer.switchDbUser(config.branchscanner.dbuser)
255+ job.run()
256+
257+ self.assertEqual(db_branch.revision_count, 5)
258+
259+
260 class TestBranchUpgradeJob(TestCaseWithFactory):
261 """Tests for `BranchUpgradeJob`."""
262
263
264=== modified file 'lib/lp/code/templates/branchmergeproposal-macros.pt'
265--- lib/lp/code/templates/branchmergeproposal-macros.pt 2009-12-18 21:31:12 +0000
266+++ lib/lp/code/templates/branchmergeproposal-macros.pt 2010-01-13 03:21:24 +0000
267@@ -55,7 +55,7 @@
268 <tal:registrant replace="structure proposal/registrant/fmt:link" />
269 <tal:date replace="proposal/date_review_requested/fmt:displaydate"/>
270 </td>
271- <td>
272+ <td style="text-align: right">
273 <tal:size replace='proposal/preview_diff/diff_lines_count' condition="proposal/preview_diff"/>
274 </td>
275 <td>
276
277=== modified file 'lib/lp/soyuz/doc/buildd-dispatching.txt'
278--- lib/lp/soyuz/doc/buildd-dispatching.txt 2009-11-13 19:34:17 +0000
279+++ lib/lp/soyuz/doc/buildd-dispatching.txt 2010-01-13 03:21:24 +0000
280@@ -128,7 +128,8 @@
281 Now let's check the build candidates which will be considered for the
282 builder 'bob':
283
284- >>> job = bob_builder.findBuildCandidate()
285+ >>> from zope.security.proxy import removeSecurityProxy
286+ >>> job = removeSecurityProxy(bob_builder)._findBuildCandidate()
287
288 The single BuildQueue found is a non-virtual pending build:
289
290@@ -157,14 +158,16 @@
291 ... 'foo.dsc', len(content), StringIO(content), 'application/dsc')
292
293 >>> sprf = build.sourcepackagerelease.files[0]
294- >>> from zope.security.proxy import removeSecurityProxy
295 >>> naked_sprf = removeSecurityProxy(sprf)
296 >>> naked_sprf.libraryfile = getUtility(ILibraryFileAliasSet)[alias_id]
297 >>> flush_database_updates()
298
299 Check the dispatching method itself:
300
301- >>> bob_builder.dispatchBuildCandidate(job)
302+ >>> dispatched_job = bob_builder.findAndStartJob()
303+ >>> job == dispatched_job
304+ True
305+
306 >>> flush_database_updates()
307
308 Verify if the job (BuildQueue) was updated appropriately:
309@@ -224,7 +227,7 @@
310 >>> bob_builder.vm_host = 'localhost.ppa'
311 >>> syncUpdate(bob_builder)
312
313- >>> job = bob_builder.findBuildCandidate()
314+ >>> job = removeSecurityProxy(bob_builder)._findBuildCandidate()
315 >>> print job
316 None
317
318@@ -245,11 +248,11 @@
319 >>> bob_builder.virtualized = True
320 >>> syncUpdate(bob_builder)
321
322- >>> job = bob_builder.findBuildCandidate()
323+ >>> job = removeSecurityProxy(bob_builder)._findBuildCandidate()
324 >>> ppa_job.id == job.id
325 True
326
327-For further details regarding IBuilder.findBuildCandidate() please see
328+For further details regarding IBuilder._findBuildCandidate() please see
329 lib/lp/soyuz/tests/test_builder.py.
330
331 Start buildd-slave to be able to dispatch jobs.
332@@ -262,7 +265,7 @@
333
334 >>> bob_builder.virtualized = False
335 >>> flush_database_updates()
336- >>> bob_builder.dispatchBuildCandidate(ppa_job)
337+ >>> removeSecurityProxy(bob_builder)._dispatchBuildCandidate(ppa_job)
338 Traceback (most recent call last):
339 ...
340 AssertionError: Attempt to build non-virtual item on a virtual builder.
341@@ -273,7 +276,10 @@
342 >>> bob_builder.virtualized = True
343 >>> flush_database_updates()
344
345- >>> bob_builder.dispatchBuildCandidate(ppa_job)
346+ >>> dispatched_job = bob_builder.findAndStartJob()
347+ >>> ppa_job == dispatched_job
348+ True
349+
350 >>> flush_database_updates()
351
352 PPA job is building.
353@@ -328,7 +334,7 @@
354 implementation.
355
356 >>> BuilddSlaveTestSetup().setUp()
357- >>> bob_builder.dispatchBuildCandidate(sec_job)
358+ >>> removeSecurityProxy(bob_builder)._dispatchBuildCandidate(sec_job)
359 Traceback (most recent call last):
360 ...
361 AssertionError: Soyuz is not yet capable of building SECURITY uploads.
362@@ -336,7 +342,7 @@
363
364 To solve this problem temporarily until we start building security
365 uploads, we will mark builds targeted to the SECURITY pocket as
366-FAILEDTOBUILD during the findBuildCandidate look-up.
367+FAILEDTOBUILD during the _findBuildCandidate look-up.
368
369 We will also create another build candidate in breezy-autotest/i386 to
370 check if legitimate pending candidates will remain valid.
371@@ -360,7 +366,7 @@
372
373 >>> new_pub = old_pub.copyTo(
374 ... pending_build.distroseries, old_pub.pocket, pending_build.archive)
375- >>> candidate = bob_builder.findBuildCandidate()
376+ >>> candidate = removeSecurityProxy(bob_builder)._findBuildCandidate()
377 >>> flush_database_updates()
378 >>> candidate.id == pending_job.id
379 True
380
381=== modified file 'lib/lp/soyuz/doc/buildd-slavescanner.txt'
382--- lib/lp/soyuz/doc/buildd-slavescanner.txt 2010-01-05 16:30:29 +0000
383+++ lib/lp/soyuz/doc/buildd-slavescanner.txt 2010-01-13 03:21:24 +0000
384@@ -754,11 +754,8 @@
385
386 == Build Dispatching ==
387
388-Build dispatching can be entirely done via IBuilder content class via
389-the following API:
390-
391- * findCandidate: returns a suitable BuildQueue candidate
392- * dispatchBuildCandidate: dispatch a build for a given candidate.
393+Build dispatching can be entirely done via IBuilder content class
394+using the findAndStartJob method.
395
396 We will use SoyuzTestPublisher to simulate the required context in the
397 next tests. Let's initialise it.
398@@ -805,14 +802,14 @@
399 superseded source package releases in the queue and marks the
400 corresponding build record as SUPERSEDED.
401
402- >>> old_candidate = a_builder.findBuildCandidate()
403+ >>> old_candidate = removeSecurityProxy(a_builder)._findBuildCandidate()
404 >>> build = getUtility(IBuildSet).getByQueueEntry(old_candidate)
405 >>> print build.buildstate.name
406 NEEDSBUILD
407
408 The 'candidate' is constant until we dispatch it.
409
410- >>> new_candidate = a_builder.findBuildCandidate()
411+ >>> new_candidate = removeSecurityProxy(a_builder)._findBuildCandidate()
412 >>> new_candidate.id == old_candidate.id
413 True
414
415@@ -820,7 +817,7 @@
416 whether the candidate will still be found.
417
418 >>> build.archive.enabled = False
419- >>> new_candidate = a_builder.findBuildCandidate()
420+ >>> new_candidate = removeSecurityProxy(a_builder)._findBuildCandidate()
421 >>> new_candidate is None
422 True
423
424@@ -829,7 +826,7 @@
425 candidate will be found again.
426
427 >>> build.archive.enabled = True
428- >>> new_candidate = a_builder.findBuildCandidate()
429+ >>> new_candidate = removeSecurityProxy(a_builder)._findBuildCandidate()
430 >>> new_candidate.id == old_candidate.id
431 True
432
433@@ -852,7 +849,7 @@
434
435 Now, there we have another build candidate.
436
437- >>> new_candidate = a_builder.findBuildCandidate()
438+ >>> new_candidate = removeSecurityProxy(a_builder)._findBuildCandidate()
439 >>> new_candidate.id != old_candidate.id
440 True
441
442@@ -882,9 +879,10 @@
443
444 Let's try to find a new build candidate:
445
446- >>> another_candidate = a_builder.findBuildCandidate()
447+ >>> another_candidate = removeSecurityProxy(
448+ ... a_builder)._findBuildCandidate()
449
450-Since there are no more candidates at all, findBuildCandidate()
451+Since there are no more candidates at all, _findBuildCandidate()
452 returned None:
453
454 >>> print another_candidate
455@@ -897,7 +895,8 @@
456 >>> commit()
457 >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
458
459- >>> another_candidate = a_builder.findBuildCandidate()
460+ >>> another_candidate = removeSecurityProxy(
461+ ... a_builder)._findBuildCandidate()
462 >>> another_candidate.id == new_candidate.id
463 True
464
465@@ -921,7 +920,8 @@
466 >>> print build.buildstate.name
467 NEEDSBUILD
468
469- >>> another_candidate = a_builder.findBuildCandidate()
470+ >>> another_candidate = removeSecurityProxy(
471+ ... a_builder)._findBuildCandidate()
472 >>> print another_candidate
473 None
474
475@@ -954,7 +954,7 @@
476 >>> a_builder.is_available
477 True
478 >>> candidate = a_build.createBuildQueueEntry()
479- >>> a_builder.dispatchBuildCandidate(candidate)
480+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
481 ensurepresent called, url=...
482 ensurepresent called,
483 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
484@@ -982,7 +982,7 @@
485 >>> a_builder.is_available
486 True
487 >>> candidate = a_build.createBuildQueueEntry()
488- >>> a_builder.dispatchBuildCandidate(candidate)
489+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
490 ensurepresent called, url=...
491 ensurepresent called,
492 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
493@@ -1034,7 +1034,7 @@
494 So, at moment, partner archive is still not relevant for builds in
495 hoary/i386. It's not passed to the builder.
496
497- >>> a_builder.dispatchBuildCandidate(candidate)
498+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
499 ensurepresent called, url=...
500 ensurepresent called,
501 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
502@@ -1084,7 +1084,8 @@
503 binary in hoary/i386, the partner archive gets included in the builder
504 sources_list.
505
506- >>> a_builder.dispatchBuildCandidate(partner_candidate)
507+ >>> removeSecurityProxy(
508+ ... a_builder)._dispatchBuildCandidate(partner_candidate)
509 ensurepresent called, url=...
510 ensurepresent called, url=http://localhost:58000/.../foo_666.dsc
511 OkSlave BUILDING
512@@ -1126,7 +1127,7 @@
513 >>> create_binary_publication_for(
514 ... cprov_archive, hoary, PackagePublishingStatus.PUBLISHED)
515
516- >>> a_builder.dispatchBuildCandidate(candidate)
517+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
518 ensurepresent called, url=...
519 ensurepresent called,
520 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
521@@ -1183,7 +1184,7 @@
522
523 This is so that the mangling tools will run over the built packages.
524
525- >>> a_builder.dispatchBuildCandidate(candidate)
526+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
527 ensurepresent called, url=...
528 ensurepresent called,
529 url=http://private-ppa.launchpad.dev/cprov/ppa/ubuntu/pool/main/m/mozilla-firefox/firefox_0.9.2.orig.tar.gz
530@@ -1223,7 +1224,7 @@
531 >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
532 >>> login(ANONYMOUS)
533
534- >>> a_builder.dispatchBuildCandidate(candidate)
535+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
536 ensurepresent called, ...
537 ...
538 Ogre-component: main
539@@ -1302,7 +1303,7 @@
540 >>> setupBuildQueue(candidate, a_builder)
541 >>> last_stub_mail_count = len(stub.test_emails)
542
543- >>> a_builder.dispatchBuildCandidate(candidate)
544+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
545 ensurepresent called, url=...
546 ensurepresent called,
547 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
548@@ -1331,7 +1332,7 @@
549 >>> create_binary_publication_for(
550 ... mark_archive, hoary, PackagePublishingStatus.PUBLISHED)
551
552- >>> a_builder.dispatchBuildCandidate(candidate)
553+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
554 ensurepresent called, url=...
555 ensurepresent called,
556 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
557@@ -1376,7 +1377,7 @@
558
559 >>> hoary_i386.distroseries.status.name
560 'DEVELOPMENT'
561- >>> a_builder.dispatchBuildCandidate(updates_bqItem)
562+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(updates_bqItem)
563 Traceback (most recent call last):
564 ...
565 AssertionError: i386 build of evolution 1.0 in ubuntu hoary UPDATES (...) can not be built for pocket UPDATES: invalid pocket due to the series status of hoary.
566@@ -1401,7 +1402,7 @@
567 >>> removeSecurityProxy(build).pocket = (
568 ... PackagePublishingPocket.UPDATES)
569 >>> last_stub_mail_count = len(stub.test_emails)
570- >>> a_builder.dispatchBuildCandidate(bqItem3)
571+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
572 ensurepresent called, url=...
573 ensurepresent called,
574 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
575@@ -1423,7 +1424,7 @@
576 >>> removeSecurityProxy(build).pocket = (
577 ... PackagePublishingPocket.PROPOSED)
578 >>> last_stub_mail_count = len(stub.test_emails)
579- >>> a_builder.dispatchBuildCandidate(bqItem3)
580+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
581 ensurepresent called, url=...
582 ensurepresent called,
583 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
584@@ -1446,7 +1447,7 @@
585 >>> removeSecurityProxy(build).pocket = (
586 ... PackagePublishingPocket.BACKPORTS)
587 >>> last_stub_mail_count = len(stub.test_emails)
588- >>> a_builder.dispatchBuildCandidate(bqItem3)
589+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
590 ensurepresent called, url=...
591 ensurepresent called,
592 url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
593@@ -1477,13 +1478,13 @@
594 because Embargoed-Archives and Restricted-UI implementations are not
595 yet ready.
596
597- >>> a_builder.dispatchBuildCandidate(bqItem3)
598+ >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
599 Traceback (most recent call last):
600 ...
601 AssertionError: Soyuz is not yet capable of building SECURITY uploads.
602
603-Builds for security pocket are marked as FAILEDTOBUILD inside
604-findBuildCandidate method, see doc/buildd-dispatching.txt
605+Builds for security pocket are marked as FAILEDTOBUILD inside the
606+_findBuildCandidate() method, see doc/buildd-dispatching.txt
607
608
609 == Builder Status Handler ==
610
611=== modified file 'lib/lp/soyuz/interfaces/builder.py'
612--- lib/lp/soyuz/interfaces/builder.py 2009-12-03 14:38:48 +0000
613+++ lib/lp/soyuz/interfaces/builder.py 2010-01-13 03:21:24 +0000
614@@ -242,24 +242,6 @@
615 :return: A librarian file alias.
616 """
617
618- def findBuildCandidate():
619- """Return the candidate for building.
620-
621- The pending BuildQueue item with the highest score for this builder
622- ProcessorFamily or None if no candidate is available.
623-
624- For public PPA builds, subsequent builds for a given ppa and
625- architecture will not be returned until the current build for
626- the ppa and architecture is finished.
627- """
628-
629- def dispatchBuildCandidate(candidate):
630- """Dispatch the given job to this builder.
631-
632- This method can only be executed in the builddmaster machine, since
633- it will actually issues the XMLRPC call to the buildd-slave.
634- """
635-
636 def handleTimeout(logger, error_message):
637 """Handle buildd slave communication timeout situations.
638
639@@ -274,6 +256,14 @@
640 :param error_message: The error message to be used for logging.
641 """
642
643+ def findAndStartJob(buildd_slave=None):
644+ """Find a job to run and send it to the buildd slave.
645+
646+ :param buildd_slave: An optional buildd slave that this builder should
647+ talk to.
648+ :return: the `IBuildQueue` instance found or None if no job was found.
649+ """
650+
651
652 class IBuilderSet(Interface):
653 """Collections of builders.
654
655=== modified file 'lib/lp/soyuz/interfaces/publishing.py'
656--- lib/lp/soyuz/interfaces/publishing.py 2010-01-05 23:32:19 +0000
657+++ lib/lp/soyuz/interfaces/publishing.py 2010-01-13 03:21:24 +0000
658@@ -29,7 +29,7 @@
659 'name_priority_map',
660 ]
661
662-from zope.schema import Bool, Choice, Datetime, Int, List, TextLine, Text
663+from zope.schema import Bool, Choice, Datetime, Int, TextLine, Text
664 from zope.interface import Interface, Attribute
665 from lazr.enum import DBEnumeratedType, DBItem
666
667@@ -534,20 +534,6 @@
668 title=_("Source Package Version"),
669 required=False, readonly=True))
670
671- source_file_urls = exported(
672- List(
673- value_type=Text(),
674- title=_("Source File URLs"),
675- description=_("URL list for this source publication's "
676- "files from the source upload.")))
677-
678- binary_file_urls = exported(
679- List(
680- value_type=Text(),
681- title=_("Binary File URLs"),
682- description=_("URL list for this source publication's "
683- "files resulting from the build.")))
684-
685 package_creator = exported(
686 Reference(
687 IPerson,
688@@ -690,6 +676,21 @@
689 }
690 """
691
692+ @export_read_operation()
693+ def sourceFileUrls():
694+ """URLs for this source publication's uploaded source files.
695+
696+ :return: A collection of URLs for this source.
697+ """
698+
699+ @export_read_operation()
700+ def binaryFileUrls():
701+ """URLs for this source publication's binary files.
702+
703+ :return: A collection of URLs for this source.
704+ """
705+
706+
707 #
708 # Binary package publishing
709 #
710
711=== modified file 'lib/lp/soyuz/model/builder.py'
712--- lib/lp/soyuz/model/builder.py 2009-12-24 06:57:25 +0000
713+++ lib/lp/soyuz/model/builder.py 2010-01-13 03:21:24 +0000
714@@ -386,9 +386,9 @@
715 return True
716
717 # XXX cprov 20071116: It should become part of the public
718- # findBuildCandidate once we start to detect superseded builds
719+ # _findBuildCandidate once we start to detect superseded builds
720 # at build creation time.
721- def _findBuildCandidate(self):
722+ def _findBinaryBuildCandidate(self):
723 """Return the highest priority build candidate for this builder.
724
725 Returns a pending IBuildQueue record queued for this builder
726@@ -487,10 +487,21 @@
727 logger = logging.getLogger('slave-scanner')
728 return logger
729
730- def findBuildCandidate(self):
731- """See `IBuilder`."""
732+ def _findBuildCandidate(self):
733+ """Find a candidate job for dispatch to an idle buildd slave.
734+
735+ The pending BuildQueue item with the highest score for this builder
736+ ProcessorFamily or None if no candidate is available.
737+
738+ For public PPA builds, subsequent builds for a given ppa and
739+ architecture will not be returned until the current build for
740+ the ppa and architecture is finished.
741+
742+ :return: A binary build candidate job.
743+ """
744+
745 logger = self._getSlaveScannerLogger()
746- candidate = self._findBuildCandidate()
747+ candidate = self._findBinaryBuildCandidate()
748
749 # Mark build records targeted to old source versions as SUPERSEDED
750 # and build records target to SECURITY pocket as FAILEDTOBUILD.
751@@ -507,7 +518,7 @@
752 % (build.id, candidate.id))
753 build.buildstate = BuildStatus.FAILEDTOBUILD
754 candidate.destroySelf()
755- candidate = self._findBuildCandidate()
756+ candidate = self._findBinaryBuildCandidate()
757 continue
758
759 publication = build.current_source_publication
760@@ -520,7 +531,7 @@
761 % (build.id, candidate.id))
762 build.buildstate = BuildStatus.SUPERSEDED
763 candidate.destroySelf()
764- candidate = self._findBuildCandidate()
765+ candidate = self._findBinaryBuildCandidate()
766 continue
767
768 return candidate
769@@ -528,8 +539,14 @@
770 # No candidate was found.
771 return None
772
773- def dispatchBuildCandidate(self, candidate):
774- """See `IBuilder`."""
775+ def _dispatchBuildCandidate(self, candidate):
776+ """Dispatch the pending job to the associated buildd slave.
777+
778+ This method can only be executed in the builddmaster machine, since
779+ it will actually issues the XMLRPC call to the buildd-slave.
780+
781+ :param candidate: The job to dispatch.
782+ """
783 logger = self._getSlaveScannerLogger()
784 try:
785 self.startBuild(candidate, logger)
786@@ -563,6 +580,21 @@
787 exc_info=True)
788 self.failbuilder(error_message)
789
790+ def findAndStartJob(self, buildd_slave=None):
791+ """See IBuilder."""
792+ logger = self._getSlaveScannerLogger()
793+ candidate = self._findBuildCandidate()
794+
795+ if candidate is None:
796+ logger.debug("No build candidates available for builder.")
797+ return None
798+
799+ if buildd_slave is not None:
800+ self.setSlaveForTesting(buildd_slave)
801+
802+ self._dispatchBuildCandidate(candidate)
803+ return candidate
804+
805
806 class BuilderSet(object):
807 """See IBuilderSet"""
808
809=== modified file 'lib/lp/soyuz/model/publishing.py'
810--- lib/lp/soyuz/model/publishing.py 2010-01-05 19:56:35 +0000
811+++ lib/lp/soyuz/model/publishing.py 2010-01-13 03:21:24 +0000
812@@ -865,24 +865,20 @@
813 return [
814 ProxiedLibraryFileAlias(file, parent).http_url for file in files]
815
816- @property
817- def source_file_urls(self):
818+ def sourceFileUrls(self):
819 """See `ISourcePackagePublishingHistory`."""
820 source_urls = self._proxied_urls(
821 [file.libraryfile for file in self.sourcepackagerelease.files],
822 self.archive)
823-
824 return source_urls
825
826- @property
827- def binary_file_urls(self):
828+ def binaryFileUrls(self):
829 """See `ISourcePackagePublishingHistory`."""
830 publishing_set = getUtility(IPublishingSet)
831 binaries = publishing_set.getBinaryFilesForSources(
832 self).config(distinct=True)
833 binary_urls = self._proxied_urls(
834 [binary for _source, binary, _content in binaries], self.archive)
835-
836 return binary_urls
837
838
839
840=== modified file 'lib/lp/soyuz/scripts/buildd.py'
841--- lib/lp/soyuz/scripts/buildd.py 2009-10-26 18:40:04 +0000
842+++ lib/lp/soyuz/scripts/buildd.py 2010-01-13 03:21:24 +0000
843@@ -201,12 +201,10 @@
844 if not builder.is_available:
845 self.logger.warn('builder is not available. Ignored.')
846 continue
847- candidate = builder.findBuildCandidate()
848+
849+ candidate = builder.findAndStartJob()
850 if candidate is None:
851- self.logger.debug(
852- "No candidates available for builder.")
853 continue
854- builder.dispatchBuildCandidate(candidate)
855 self.txn.commit()
856
857 self.logger.info("Slave Scan Process Finished.")
858
859=== modified file 'lib/lp/soyuz/stories/webservice/xx-source-package-publishing.txt'
860--- lib/lp/soyuz/stories/webservice/xx-source-package-publishing.txt 2010-01-05 23:32:19 +0000
861+++ lib/lp/soyuz/stories/webservice/xx-source-package-publishing.txt 2010-01-13 03:21:24 +0000
862@@ -111,7 +111,6 @@
863 >>> from lazr.restful.testing.webservice import pprint_entry
864 >>> pprint_entry(pubs['entries'][0])
865 archive_link: u'http://.../~cprov/+archive/ppa'
866- binary_file_urls: []
867 component_name: u'main'
868 date_created: ...
869 date_made_pending: None
870@@ -130,7 +129,6 @@
871 scheduled_deletion_date: None
872 section_name: u'base'
873 self_link: u'http://.../~cprov/+archive/ppa/+sourcepub/...'
874- source_file_urls: [u'http://launchpad.dev/~cprov/+archive/ppa/+files/testwebservice_666.dsc']
875 source_package_name: u'testwebservice'
876 source_package_version: u'666'
877 status: u'Pending'
878@@ -342,23 +340,23 @@
879
880 >>> pubs = webservice.named_get(
881 ... cprov_archive['self_link'], 'getPublishedSources').jsonBody()
882- >>> def print_source_urls(pubs):
883- ... for source_file_urls in sorted(
884- ... entry['source_file_urls'] for entry in pubs['entries']):
885- ... print source_file_urls
886- >>> print_source_urls(pubs)
887- []
888+ >>> for pub_link in sorted(
889+ ... entry['self_link'] for entry in pubs['entries']):
890+ ... source_urls = webservice.named_get(
891+ ... pub_link, 'sourceFileUrls').jsonBody()
892+ ... print source_urls
893+ [u'http://launchpad.dev/~cprov/+archive/ppa/+files/foobar-1.0.dsc']
894 [u'http://launchpad.dev/~cprov/+archive/ppa/+files/firefox_0.9.2.orig.tar.gz', u'http://launchpad.dev/~cprov/+archive/ppa/+files/iceweasel-1.0.dsc']
895- [u'http://launchpad.dev/~cprov/+archive/ppa/+files/foobar-1.0.dsc']
896+ []
897 [u'http://launchpad.dev/~cprov/+archive/ppa/+files/testwebservice_666.dsc']
898
899- >>> def print_binary_urls(pubs):
900- ... for binary_file_urls in sorted(
901- ... entry['binary_file_urls'] for entry in pubs['entries']):
902- ... print binary_file_urls
903- >>> print_binary_urls(pubs)
904- []
905- []
906+ >>> for pub_link in sorted(
907+ ... entry['self_link'] for entry in pubs['entries']):
908+ ... binary_urls = webservice.named_get(
909+ ... pub_link, 'binaryFileUrls').jsonBody()
910+ ... print binary_urls
911 []
912 [u'http://launchpad.dev/~cprov/+archive/ppa/+files/mozilla-firefox_0.9_i386.deb']
913+ []
914+ []
915
916
917=== modified file 'lib/lp/soyuz/tests/test_builder.py'
918--- lib/lp/soyuz/tests/test_builder.py 2009-12-02 15:18:46 +0000
919+++ lib/lp/soyuz/tests/test_builder.py 2010-01-13 03:21:24 +0000
920@@ -6,10 +6,11 @@
921 import unittest
922
923 from zope.component import getUtility
924+from zope.security.proxy import removeSecurityProxy
925
926 from canonical.testing import LaunchpadZopelessLayer
927 from lp.buildmaster.interfaces.buildfarmjobbehavior import (
928- BuildBehaviorMismatch, IBuildFarmJobBehavior)
929+ IBuildFarmJobBehavior)
930 from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior
931 from lp.soyuz.interfaces.archive import ArchivePurpose
932 from lp.soyuz.interfaces.build import BuildStatus, IBuildSet
933@@ -34,13 +35,13 @@
934 # Create some i386 builders ready to build PPA builds. Two
935 # already exist in sampledata so we'll use those first.
936 self.builder1 = getUtility(IBuilderSet)['bob']
937- self.builder2 = getUtility(IBuilderSet)['frog']
938+ self.frog_builder = getUtility(IBuilderSet)['frog']
939 self.builder3 = self.factory.makeBuilder(name='builder3')
940 self.builder4 = self.factory.makeBuilder(name='builder4')
941 self.builder5 = self.factory.makeBuilder(name='builder5')
942 self.builders = [
943 self.builder1,
944- self.builder2,
945+ self.frog_builder,
946 self.builder3,
947 self.builder4,
948 self.builder5,
949@@ -81,7 +82,8 @@
950 # there's only one builder available.
951
952 # Asking frog to find a candidate should give us the joesppa build.
953- next_job = self.frog_builder.findBuildCandidate()
954+ next_job = removeSecurityProxy(
955+ self.frog_builder)._findBuildCandidate()
956 build = getUtility(IBuildSet).getByQueueEntry(next_job)
957 self.assertEqual('joesppa', build.archive.name)
958
959@@ -89,7 +91,8 @@
960 # returned.
961 self.bob_builder.builderok = False
962 self.bob_builder.manual = False
963- next_job = self.frog_builder.findBuildCandidate()
964+ next_job = removeSecurityProxy(
965+ self.frog_builder)._findBuildCandidate()
966 build = getUtility(IBuildSet).getByQueueEntry(next_job)
967 self.assertEqual('joesppa', build.archive.name)
968
969@@ -162,7 +165,7 @@
970 def test_findBuildCandidate_first_build_started(self):
971 # A PPA cannot start a build if it would use 80% or more of the
972 # builders.
973- next_job = self.builder4.findBuildCandidate()
974+ next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
975 build = getUtility(IBuildSet).getByQueueEntry(next_job)
976 self.failIfEqual('joesppa', build.archive.name)
977
978@@ -170,7 +173,7 @@
979 # When joe's first ppa build finishes, his fourth i386 build
980 # will be the next build candidate.
981 self.joe_builds[0].buildstate = BuildStatus.FAILEDTOBUILD
982- next_job = self.builder4.findBuildCandidate()
983+ next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
984 build = getUtility(IBuildSet).getByQueueEntry(next_job)
985 self.failUnlessEqual('joesppa', build.archive.name)
986
987@@ -179,7 +182,7 @@
988 # for the one architecture.
989 self.ppa_joe.private = True
990 self.ppa_joe.buildd_secret = 'sekrit'
991- next_job = self.builder4.findBuildCandidate()
992+ next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
993 build = getUtility(IBuildSet).getByQueueEntry(next_job)
994 self.failUnlessEqual('joesppa', build.archive.name)
995
996@@ -205,7 +208,8 @@
997 # Normal archives are not restricted to serial builds per
998 # arch.
999
1000- next_job = self.builder2.findBuildCandidate()
1001+ next_job = removeSecurityProxy(
1002+ self.frog_builder)._findBuildCandidate()
1003 build = getUtility(IBuildSet).getByQueueEntry(next_job)
1004 self.failUnlessEqual('primary', build.archive.name)
1005 self.failUnlessEqual('gedit', build.sourcepackagerelease.name)
1006@@ -213,8 +217,9 @@
1007 # Now even if we set the build building, we'll still get the
1008 # second non-ppa build for the same archive as the next candidate.
1009 build.buildstate = BuildStatus.BUILDING
1010- build.builder = self.builder2
1011- next_job = self.builder2.findBuildCandidate()
1012+ build.builder = self.frog_builder
1013+ next_job = removeSecurityProxy(
1014+ self.frog_builder)._findBuildCandidate()
1015 build = getUtility(IBuildSet).getByQueueEntry(next_job)
1016 self.failUnlessEqual('primary', build.archive.name)
1017 self.failUnlessEqual('firefox', build.sourcepackagerelease.name)
1018
1019=== modified file 'lib/lp/translations/configure.zcml'
1020--- lib/lp/translations/configure.zcml 2009-12-28 22:58:18 +0000
1021+++ lib/lp/translations/configure.zcml 2010-01-13 03:21:24 +0000
1022@@ -572,6 +572,35 @@
1023 interface="lp.translations.interfaces.poexportrequest.IPOExportRequestSet"/>
1024 </securedutility>
1025
1026+ <!-- TranslationTemplateBuildJob -->
1027+ <class
1028+ class="lp.translations.model.translationtemplatesbuildjob.TranslationTemplatesBuildJob">
1029+ <allow
1030+ interface="lp.translations.interfaces.translationtemplatesbuildjob.ITranslationTemplatesBuildJob"/>
1031+ </class>
1032+ <securedutility
1033+ component="lp.translations.model.translationtemplatesbuildjob.TranslationTemplatesBuildJob"
1034+ provides="lp.translations.interfaces.translationtemplatesbuildjob.ITranslationTemplatesBuildJobSource">
1035+ <allow interface="lp.translations.interfaces.translationtemplatesbuildjob.ITranslationTemplatesBuildJobSource"/>
1036+ </securedutility>
1037+ <utility
1038+ component="lp.translations.model.translationtemplatesbuildjob.TranslationTemplatesBuildJob"
1039+ provides="lp.translations.interfaces.translationtemplatesbuildjob.ITranslationTemplatesBuildJob"
1040+ name="TRANSLATIONTEMPLATESBUILD"/>
1041+
1042+ <!-- TranslationTemplateBuildBehavior -->
1043+ <class
1044+ class="lp.translations.model.translationtemplatesbuildbehavior.TranslationTemplatesBuildBehavior">
1045+ <allow
1046+ interface="lp.buildmaster.interfaces.buildfarmjobbehavior.IBuildFarmJobBehavior"/>
1047+ </class>
1048+
1049+ <adapter
1050+ provides="lp.buildmaster.interfaces.buildfarmjobbehavior.IBuildFarmJobBehavior"
1051+ for="lp.translations.model.translationtemplatesbuildjob.TranslationTemplatesBuildJob"
1052+ factory="lp.translations.model.translationtemplatesbuildbehavior.TranslationTemplatesBuildBehavior"
1053+ />
1054+
1055 <webservice:register module="lp.translations.interfaces.webservice" />
1056
1057 </configure>
1058
1059=== added file 'lib/lp/translations/interfaces/translationtemplatesbuildjob.py'
1060--- lib/lp/translations/interfaces/translationtemplatesbuildjob.py 1970-01-01 00:00:00 +0000
1061+++ lib/lp/translations/interfaces/translationtemplatesbuildjob.py 2010-01-13 03:21:24 +0000
1062@@ -0,0 +1,37 @@
1063+# Copyright 2010 Canonical Ltd. This software is licensed under the
1064+# GNU Affero General Public License version 3 (see the file LICENSE).
1065+
1066+# pylint: disable-msg=E0213
1067+
1068+__metaclass__ = type
1069+
1070+__all__ = [
1071+ 'ITranslationTemplatesBuildJob',
1072+ 'ITranslationTemplatesBuildJobSource',
1073+ ]
1074+
1075+from zope.interface import Interface
1076+
1077+from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
1078+from lp.code.interfaces.branchjob import IBranchJob
1079+
1080+
1081+class ITranslationTemplatesBuildJob(IBranchJob, IBuildFarmJob):
1082+ """Build-farm job type for generating translation templates."""
1083+
1084+
1085+class ITranslationTemplatesBuildJobSource(Interface):
1086+ """Container for `ITranslationTemplatesBuildJob`s."""
1087+
1088+ def create(branch):
1089+ """Create new `ITranslationTemplatesBuildJob`.
1090+
1091+ Also creates the matching `IBuildQueue` and `IJob`.
1092+
1093+ :param branch: A `Branch` that this job will check out and
1094+ generate templates for.
1095+ :return: A new `ITranslationTemplatesBuildJob`.
1096+ """
1097+
1098+ def getForJob(job):
1099+ """Find `ITranslationTemplatesBuildJob` matching given `Job`."""
1100
1101=== added file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py'
1102--- lib/lp/translations/model/translationtemplatesbuildbehavior.py 1970-01-01 00:00:00 +0000
1103+++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-01-13 03:21:24 +0000
1104@@ -0,0 +1,79 @@
1105+# Copyright 2010 Canonical Ltd. This software is licensed under the
1106+# GNU Affero General Public License version 3 (see the file LICENSE).
1107+
1108+"""An `IBuildFarmJobBehavior` for `TranslationTemplatesBuildJob`.
1109+
1110+Dispatches translation template build jobs to build-farm slaves.
1111+"""
1112+
1113+__metaclass__ = type
1114+__all__ = [
1115+ 'TranslationTemplatesBuildBehavior',
1116+ ]
1117+
1118+import socket
1119+import xmlrpclib
1120+
1121+from zope.component import getUtility
1122+from zope.interface import implements
1123+
1124+from canonical.launchpad.interfaces import ILaunchpadCelebrities
1125+
1126+from lp.buildmaster.interfaces.buildfarmjobbehavior import (
1127+ IBuildFarmJobBehavior)
1128+from lp.buildmaster.model.buildfarmjobbehavior import (
1129+ BuildFarmJobBehaviorBase)
1130+from lp.soyuz.interfaces.builder import BuildSlaveFailure
1131+from lp.translations.interfaces.translationtemplatesbuildjob import (
1132+ ITranslationTemplatesBuildJobSource)
1133+
1134+
1135+class TranslationTemplatesBuildBehavior(BuildFarmJobBehaviorBase):
1136+ """Dispatches `TranslationTemplateBuildJob`s to slaves."""
1137+ implements(IBuildFarmJobBehavior)
1138+
1139+ # Identify the type of job to the slave.
1140+ build_type = 'translation-templates'
1141+
1142+ def dispatchBuildToSlave(self, build_queue_item, logger):
1143+ """See `IBuildFarmJobBehavior`."""
1144+ # XXX JeroenVermeulen 2009-12-24 bug=500110: This method is not
1145+ # covered by tests yet. Either unify it with Soyuz code into a
1146+ # generalised method, or test it.
1147+ templatesbuildjob = self._findTranslationTemplatesBuildJob(
1148+ build_queue_item)
1149+ chroot = self._getChroot()
1150+ chroot_sha1 = chroot.content.sha1
1151+ self._builder.cacheFileOnSlave(logger, chroot)
1152+ buildid = templatesbuildjob.getName()
1153+
1154+ args = { 'branch_url': build_queue_item.branch.url }
1155+ filemap = {}
1156+
1157+ try:
1158+ status, info = self._builder.slave.build(
1159+ buildid, self.build_type, chroot_sha1, filemap, args)
1160+ except xmlrpclib.Fault, info:
1161+ # Mark builder as 'failed'.
1162+ logger.debug(
1163+ "Disabling builder: %s" % self._builder.url, exc_info=1)
1164+ self._builder.failbuilder(
1165+ "Exception (%s) when setting up to new job" % info)
1166+ raise BuildSlaveFailure
1167+ except socket.error, info:
1168+ error_message = "Exception (%s) when setting up new job" % info
1169+ self._builder.handleTimeout(logger, error_message)
1170+ raise BuildSlaveFailure
1171+
1172+ def _getChroot(self):
1173+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
1174+ return ubuntu.currentseries.nominatedarchindep.getChroot()
1175+
1176+ def _findTranslationTemplatesBuildJob(self, build_queue_item):
1177+ """Find the `TranslationTemplatesBuildJob` for a job.
1178+
1179+ :param build_queue_item: A `BuildQueue` entry.
1180+ :return: The matching `TranslationTemplatesBuildJob`.
1181+ """
1182+ jobsource = getUtility(ITranslationTemplatesBuildJobSource)
1183+ return jobsource.getForJob(build_queue_item.job)
1184
1185=== added file 'lib/lp/translations/model/translationtemplatesbuildjob.py'
1186--- lib/lp/translations/model/translationtemplatesbuildjob.py 1970-01-01 00:00:00 +0000
1187+++ lib/lp/translations/model/translationtemplatesbuildjob.py 2010-01-13 03:21:24 +0000
1188@@ -0,0 +1,85 @@
1189+# Copyright 2010 Canonical Ltd. This software is licensed under the
1190+# GNU Affero General Public License version 3 (see the file LICENSE).
1191+
1192+__metaclass__ = type
1193+__all__ = [
1194+ 'TranslationTemplatesBuildJob',
1195+ ]
1196+
1197+import re
1198+from datetime import timedelta
1199+
1200+from zope.component import getUtility
1201+from zope.interface import classProvides, implements
1202+
1203+from canonical.launchpad.webapp.interfaces import (
1204+ DEFAULT_FLAVOR, IStoreSelector, MAIN_STORE, MASTER_FLAVOR)
1205+
1206+from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
1207+from lp.buildmaster.model.buildfarmjob import BuildFarmJob
1208+from lp.code.model.branchjob import BranchJob, BranchJobDerived, BranchJobType
1209+from lp.soyuz.model.buildqueue import BuildQueue
1210+from lp.translations.interfaces.translationtemplatesbuildjob import (
1211+ ITranslationTemplatesBuildJob, ITranslationTemplatesBuildJobSource)
1212+
1213+
1214+class TranslationTemplatesBuildJob(BranchJobDerived, BuildFarmJob):
1215+ """An `IBuildFarmJob` implementation that generates templates.
1216+
1217+ Implementation-wise, this is actually a `BranchJob`.
1218+ """
1219+ implements(ITranslationTemplatesBuildJob)
1220+ classProvides(ITranslationTemplatesBuildJobSource)
1221+
1222+ duration_estimate = timedelta(seconds=10)
1223+
1224+ unsafe_chars = '[^a-zA-Z0-9_+-]'
1225+
1226+ def __init__(self, branch_job):
1227+ super(TranslationTemplatesBuildJob, self).__init__(branch_job)
1228+
1229+ def score(self):
1230+ """See `IBuildFarmJob`."""
1231+ # Hard-code score for now; anything other than 1000 is probably
1232+ # inappropriate.
1233+ return 1000
1234+
1235+ def getLogFileName(self):
1236+ """See `IBuildFarmJob`."""
1237+ sanitized_name = re.sub(self.unsafe_chars, '_', self.getName())
1238+ return "translationtemplates_%s" % sanitized_name
1239+
1240+ def getName(self):
1241+ """See `IBuildFarmJob`."""
1242+ return '%s-%d' % (self.branch.name, self.job.id)
1243+
1244+ @classmethod
1245+ def create(cls, branch):
1246+ """See `ITranslationTemplatesBuildJobSource`."""
1247+ store = getUtility(IStoreSelector).get(MAIN_STORE, MASTER_FLAVOR)
1248+
1249+ # We don't have any JSON metadata for this BranchJob type.
1250+ metadata = {}
1251+ branch_job = BranchJob(
1252+ branch, BranchJobType.TRANSLATION_TEMPLATES_BUILD, metadata)
1253+ store.add(branch_job)
1254+ specific_job = TranslationTemplatesBuildJob(branch_job)
1255+
1256+ duration_estimate = cls.duration_estimate
1257+ build_queue_entry = BuildQueue(
1258+ estimated_duration=duration_estimate,
1259+ job_type=BuildFarmJobType.TRANSLATIONTEMPLATESBUILD,
1260+ job=specific_job.job.id)
1261+ store.add(build_queue_entry)
1262+
1263+ return specific_job
1264+
1265+ @classmethod
1266+ def getForJob(cls, job):
1267+ """See `ITranslationTemplatesBuildJobSource`."""
1268+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
1269+ branch_job = store.find(BranchJob, BranchJob.job == job).one()
1270+ if branch_job is None:
1271+ return None
1272+ else:
1273+ return cls(branch_job)
1274
1275=== added file 'lib/lp/translations/tests/test_translationtemplatesbuildjob.py'
1276--- lib/lp/translations/tests/test_translationtemplatesbuildjob.py 1970-01-01 00:00:00 +0000
1277+++ lib/lp/translations/tests/test_translationtemplatesbuildjob.py 2010-01-13 03:21:24 +0000
1278@@ -0,0 +1,132 @@
1279+# Copyright 2010 Canonical Ltd. This software is licensed under the
1280+# GNU Affero General Public License version 3 (see the file LICENSE).
1281+
1282+__metaclass__ = type
1283+
1284+from unittest import TestLoader
1285+
1286+from zope.component import getUtility
1287+from zope.security.proxy import removeSecurityProxy
1288+
1289+from canonical.launchpad.interfaces import (
1290+ ILaunchpadCelebrities, ILibraryFileAliasSet)
1291+from canonical.launchpad.webapp.testing import verifyObject
1292+from canonical.testing import ZopelessDatabaseLayer
1293+
1294+from lp.testing import TestCaseWithFactory
1295+
1296+from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
1297+from lp.buildmaster.interfaces.buildfarmjobbehavior import (
1298+ IBuildFarmJobBehavior)
1299+from lp.services.job.model.job import Job
1300+from lp.soyuz.interfaces.buildqueue import IBuildQueueSet
1301+from lp.soyuz.model.buildqueue import BuildQueue
1302+from lp.translations.interfaces.translationtemplatesbuildjob import (
1303+ ITranslationTemplatesBuildJob, ITranslationTemplatesBuildJobSource)
1304+
1305+
1306+def get_job_id(job):
1307+ """Peek inside a `Job` and retrieve its id."""
1308+ return removeSecurityProxy(job).id
1309+
1310+
1311+class TestTranslationTemplatesBuildJob(TestCaseWithFactory):
1312+ """Test `TranslationTemplatesBuildJob`."""
1313+
1314+ layer = ZopelessDatabaseLayer
1315+
1316+ def setUp(self):
1317+ super(TestTranslationTemplatesBuildJob, self).setUp()
1318+ self.jobset = getUtility(ITranslationTemplatesBuildJobSource)
1319+ self.branch = self.factory.makeBranch()
1320+ self.specific_job = self.jobset.create(self.branch)
1321+
1322+ def test_new_TranslationTemplatesBuildJob(self):
1323+ # TranslationTemplateBuildJob implements IBuildFarmJob and
1324+ # ITranslationTemplatesBuildJob.
1325+ verifyObject(IBuildFarmJob, self.specific_job)
1326+ verifyObject(ITranslationTemplatesBuildJob, self.specific_job)
1327+
1328+ # The class also implements a utility.
1329+ verifyObject(ITranslationTemplatesBuildJobSource, self.jobset)
1330+
1331+ # Each of these jobs knows the branch it will operate on.
1332+ self.assertEqual(self.branch, self.specific_job.branch)
1333+
1334+ def test_has_Job(self):
1335+ # Associated with each TranslationTemplateBuildJob is a Job.
1336+ base_job = self.specific_job.job
1337+ self.assertIsInstance(base_job, Job)
1338+
1339+ # From a Job, the TranslationTemplatesBuildJobSource can find the
1340+ # TranslationTemplatesBuildJob back for us.
1341+ specific_job_for_base_job = removeSecurityProxy(
1342+ self.jobset.getForJob(base_job))
1343+ self.assertEqual(self.specific_job, specific_job_for_base_job)
1344+
1345+ def test_has_BuildQueue(self):
1346+ # There's also a BuildQueue item associated with the job.
1347+ queueset = getUtility(IBuildQueueSet)
1348+ job_id = get_job_id(self.specific_job.job)
1349+ buildqueue = queueset.get(job_id)
1350+
1351+ self.assertIsInstance(buildqueue, BuildQueue)
1352+ self.assertEqual(job_id, get_job_id(buildqueue.job))
1353+
1354+ def test_getName(self):
1355+ # Each job gets a unique name.
1356+ other_job = self.jobset.create(self.branch)
1357+ self.assertNotEqual(self.specific_job.getName(), other_job.getName())
1358+
1359+ def test_getLogFileName(self):
1360+ # Each job has a unique log file name.
1361+ other_job = self.jobset.create(self.branch)
1362+ self.assertNotEqual(
1363+ self.specific_job.getLogFileName(), other_job.getLogFileName())
1364+
1365+ def test_score(self):
1366+ # For now, these jobs always score themselves at 1,000. In the
1367+ # future however the scoring system is to be revisited.
1368+ self.assertEqual(1000, self.specific_job.score())
1369+
1370+
1371+class TestTranslationTemplatesBuildBehavior(TestCaseWithFactory):
1372+ """Test `TranslationTemplatesBuildBehavior`."""
1373+
1374+ layer = ZopelessDatabaseLayer
1375+
1376+ def setUp(self):
1377+ super(TestTranslationTemplatesBuildBehavior, self).setUp()
1378+ self.jobset = getUtility(ITranslationTemplatesBuildJobSource)
1379+ self.branch = self.factory.makeBranch()
1380+ self.specific_job = self.jobset.create(self.branch)
1381+ self.behavior = IBuildFarmJobBehavior(self.specific_job)
1382+
1383+ def test_getChroot(self):
1384+ # _getChroot produces the current chroot for the current Ubuntu
1385+ # release, on the nominated architecture for
1386+ # architecture-independent builds.
1387+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
1388+ current_ubuntu = ubuntu.currentseries
1389+ distroarchseries = current_ubuntu.nominatedarchindep
1390+
1391+ # Set an arbitrary chroot file.
1392+ fake_chroot_file = getUtility(ILibraryFileAliasSet)[1]
1393+ distroarchseries.addOrUpdateChroot(fake_chroot_file)
1394+
1395+ chroot = self.behavior._getChroot()
1396+
1397+ self.assertNotEqual(None, chroot)
1398+ self.assertEqual(fake_chroot_file, chroot)
1399+
1400+ def test_findTranslationTemplatesBuildJob(self):
1401+ job = self.specific_job.job
1402+ job_id = removeSecurityProxy(job).id
1403+ buildqueue = getUtility(IBuildQueueSet).get(job_id)
1404+ specific_job_for_buildqueue = removeSecurityProxy(
1405+ self.behavior._findTranslationTemplatesBuildJob(buildqueue))
1406+ self.assertEqual(self.specific_job, specific_job_for_buildqueue)
1407+
1408+
1409+def test_suite():
1410+ return TestLoader().loadTestsFromName(__name__)
1411
1412=== modified file 'utilities/sourcedeps.conf'
1413--- utilities/sourcedeps.conf 2009-12-02 02:46:05 +0000
1414+++ utilities/sourcedeps.conf 2010-01-13 03:21:24 +0000
1415@@ -8,7 +8,7 @@
1416 lpreview lp:~launchpad-pqm/bzr-lpreview/devel;revno=23
1417 mailman lp:~launchpad-pqm/mailman/2.1;revno=976
1418 old_xmlplus lp:~launchpad/dtdparser/trunk;revno=4
1419-pygettextpo lp:~launchpad/pygettextpo/trunk;revno=23
1420+pygettextpo lp:~launchpad-pqm/pygettextpo/trunk;revno=23
1421 pygpgme lp:~launchpad-pqm/pygpgme/devel;revno=48
1422 subunit lp:~launchpad-pqm/subunit/trunk;revno=61
1423 subvertpy lp:~launchpad-pqm/subvertpy/trunk;revno=2040

Subscribers

People subscribed via source and target branches

to status/vote changes: