Launchpad itself

Merge lp:~cjwatson/launchpad/build-private-bpb-immediately into lp:launchpad

  1. build-private-bpb-immediately
  2. Merge into devel
Proposed by Colin Watson
Status: Rejected
Rejected by: Colin Watson
Proposed branch: lp:~cjwatson/launchpad/build-private-bpb-immediately
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/librarian-accept-macaroon
Diff against target: 276 lines (+60/-64)
4 files modified
lib/lp/buildmaster/tests/test_builder.py (+4/-4)
lib/lp/soyuz/model/binarypackagebuild.py (+3/-27)
lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+12/-17)
lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+41/-16)
To merge this branch: bzr merge lp:~cjwatson/launchpad/build-private-bpb-immediately
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+345104@code.launchpad.net

Commit message

Dispatch private BPBs immediately, using macaroon auth for source files.

Description of the change

The prerequisite branch must be deployed to the librarian before landing this.

I haven't yet tested this beyond what's in the test suite.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) wrote :

Superseded by https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/373741.

Unmerged revisions

18632. By Colin Watson

Dispatch private BPBs immediately, using macaroon auth for source files.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
2--- lib/lp/buildmaster/tests/test_builder.py 2018-01-10 10:45:24 +0000
3+++ lib/lp/buildmaster/tests/test_builder.py 2018-05-04 17:00:50 +0000
4@@ -401,13 +401,13 @@
5 build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
6 self.assertEqual('joesppa', build.archive.name)
7
8- # If the source for the build is still pending, it won't be
9- # dispatched because the builder has to fetch the source files
10- # from the (password protected) repo area, not the librarian.
11+ # If the source for the build is still pending, it will still be
12+ # dispatched: the builder will use macaroon authentication to fetch
13+ # the source files from the librarian.
14 pub = build.current_source_publication
15 pub.status = PackagePublishingStatus.PENDING
16 candidate = removeSecurityProxy(self.builder4)._findBuildCandidate()
17- self.assertNotEqual(next_job.id, candidate.id)
18+ self.assertEqual(next_job.id, candidate.id)
19
20
21 class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
22
23=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
24--- lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
25+++ lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
26@@ -87,10 +87,7 @@
27 )
28 from lp.services.macaroons.interfaces import IMacaroonIssuer
29 from lp.soyuz.adapters.buildarch import determine_architectures_to_build
30-from lp.soyuz.enums import (
31- ArchivePurpose,
32- PackagePublishingStatus,
33- )
34+from lp.soyuz.enums import ArchivePurpose
35 from lp.soyuz.interfaces.archive import (
36 InvalidExternalDependencies,
37 validate_external_dependencies,
38@@ -1210,33 +1207,12 @@
39 @staticmethod
40 def addCandidateSelectionCriteria(processor, virtualized):
41 """See `ISpecificBuildFarmJobSource`."""
42- private_statuses = (
43- PackagePublishingStatus.PUBLISHED,
44- PackagePublishingStatus.SUPERSEDED,
45- PackagePublishingStatus.DELETED,
46- )
47 return """
48- SELECT TRUE FROM Archive, BinaryPackageBuild, DistroArchSeries
49+ SELECT TRUE FROM BinaryPackageBuild
50 WHERE
51 BinaryPackageBuild.build_farm_job = BuildQueue.build_farm_job AND
52- BinaryPackageBuild.distro_arch_series =
53- DistroArchSeries.id AND
54- BinaryPackageBuild.archive = Archive.id AND
55- ((Archive.private IS TRUE AND
56- EXISTS (
57- SELECT SourcePackagePublishingHistory.id
58- FROM SourcePackagePublishingHistory
59- WHERE
60- SourcePackagePublishingHistory.distroseries =
61- DistroArchSeries.distroseries AND
62- SourcePackagePublishingHistory.sourcepackagerelease =
63- BinaryPackageBuild.source_package_release AND
64- SourcePackagePublishingHistory.archive = Archive.id AND
65- SourcePackagePublishingHistory.status IN %s))
66- OR
67- archive.private IS FALSE) AND
68 BinaryPackageBuild.status = %s
69- """ % sqlvalues(private_statuses, BuildStatus.NEEDSBUILD)
70+ """ % sqlvalues(BuildStatus.NEEDSBUILD)
71
72 @staticmethod
73 def postprocessCandidate(job, logger):
74
75=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py'
76--- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
77+++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
78@@ -1,4 +1,4 @@
79-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
80+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
81 # GNU Affero General Public License version 3 (see the file LICENSE).
82
83 """Builder behaviour for binary package builds."""
84@@ -10,7 +10,9 @@
85 ]
86
87 from twisted.internet import defer
88+from zope.component import getUtility
89 from zope.interface import implementer
90+from zope.security.proxy import removeSecurityProxy
91
92 from lp.buildmaster.interfaces.builder import CannotBuild
93 from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
94@@ -20,16 +22,13 @@
95 BuildFarmJobBehaviourBase,
96 )
97 from lp.registry.interfaces.pocket import PackagePublishingPocket
98-from lp.services.webapp import (
99- canonical_url,
100- urlappend,
101- )
102+from lp.services.macaroons.interfaces import IMacaroonIssuer
103+from lp.services.webapp import canonical_url
104 from lp.soyuz.adapters.archivedependencies import (
105 get_primary_current_component,
106 get_sources_list_for_building,
107 )
108 from lp.soyuz.enums import ArchivePurpose
109-from lp.soyuz.model.publishing import makePoolPath
110
111
112 @implementer(IBuildFarmJobBehaviour)
113@@ -63,14 +62,9 @@
114 # Build filemap structure with the files required in this build
115 # and send them to the slave.
116 if self.build.archive.private:
117- # Builds in private archive may have restricted files that
118- # we can't obtain from the public librarian. Prepare a pool
119- # URL from which to fetch them.
120- pool_url = urlappend(
121- self.build.archive.archive_url,
122- makePoolPath(
123- self.build.source_package_release.sourcepackagename.name,
124- self.build.current_component.name))
125+ issuer = getUtility(IMacaroonIssuer, 'binary-package-build')
126+ password = removeSecurityProxy(issuer).issueMacaroon(
127+ self.build).serialize()
128 filemap = {}
129 for source_file in self.build.source_package_release.files:
130 lfa = source_file.libraryfile
131@@ -80,9 +74,10 @@
132 else:
133 filemap[lfa.filename] = {
134 'sha1': lfa.content.sha1,
135- 'url': urlappend(pool_url, lfa.filename),
136- 'username': 'buildd',
137- 'password': self.build.archive.buildd_secret}
138+ 'url': lfa.https_url,
139+ 'username': '',
140+ 'password': password,
141+ }
142 return filemap
143
144 @defer.inlineCallbacks
145
146=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
147--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
148+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
149@@ -12,8 +12,15 @@
150 import shutil
151 import tempfile
152
153+from pymacaroons import Macaroon
154 from storm.store import Store
155-from testtools.matchers import MatchesListwise
156+from testtools.matchers import (
157+ Contains,
158+ Equals,
159+ Matcher,
160+ MatchesListwise,
161+ Mismatch,
162+ )
163 from testtools.twistedsupport import AsynchronousDeferredRunTest
164 import transaction
165 from twisted.internet import defer
166@@ -21,7 +28,6 @@
167 from zope.component import getUtility
168 from zope.security.proxy import removeSecurityProxy
169
170-from lp.archivepublisher.diskpool import poolify
171 from lp.archivepublisher.interfaces.archivesigningkey import (
172 IArchiveSigningKey,
173 )
174@@ -58,6 +64,7 @@
175 from lp.services.config import config
176 from lp.services.librarian.interfaces import ILibraryFileAliasSet
177 from lp.services.log.logger import BufferLogger
178+from lp.services.macaroons.interfaces import IMacaroonIssuer
179 from lp.services.webapp import canonical_url
180 from lp.soyuz.adapters.archivedependencies import (
181 get_sources_list_for_building,
182@@ -74,6 +81,26 @@
183 from lp.testing.layers import LaunchpadZopelessLayer
184
185
186+class BinaryPackageBuildMacaroonVerifies(Matcher):
187+ """Matches if a binary-package-build macaroon passes verification."""
188+
189+ def __init__(self, build, lfa_id):
190+ self.build = build
191+ self.lfa_id = lfa_id
192+
193+ def match(self, macaroon_raw):
194+ macaroon = Macaroon.deserialize(macaroon_raw)
195+ expected_caveat = (
196+ "lp.binary-package-build %s" % removeSecurityProxy(self.build).id)
197+ mismatch = Contains(expected_caveat).match(
198+ [caveat.caveat_id for caveat in macaroon.caveats])
199+ if mismatch is not None:
200+ return mismatch
201+ issuer = getUtility(IMacaroonIssuer, "binary-package-build")
202+ if not issuer.verifyMacaroon(macaroon, self.lfa_id):
203+ return Mismatch("Macaroon does not verify")
204+
205+
206 class TestBinaryBuildPackageBehaviour(TestCaseWithFactory):
207 """Tests for the BinaryPackageBuildBehaviour.
208
209@@ -98,7 +125,7 @@
210 expected = yield self.makeExpectedInteraction(
211 builder, build, chroot, archive, archive_purpose, component,
212 extra_uploads, filemap_names)
213- self.assertEqual(expected, call_log)
214+ self.assertThat(call_log, MatchesListwise(expected))
215
216 @defer.inlineCallbacks
217 def makeExpectedInteraction(self, builder, build, chroot, archive,
218@@ -115,7 +142,7 @@
219 builder. We specify this separately from the archive because
220 sometimes the behaviour object has to give a different purpose
221 in order to trick the slave into building correctly.
222- :return: A list of the calls we expect to be made.
223+ :return: A list of matchers for the calls we expect to be made.
224 """
225 das = build.distro_arch_series
226 ds_name = das.distroseries.name
227@@ -131,8 +158,9 @@
228 extra_uploads = []
229
230 upload_logs = [
231- ('ensurepresent',) + upload
232- for upload in [(chroot.http_url, '', '')] + extra_uploads]
233+ MatchesListwise((Equals('ensurepresent'),) + upload)
234+ for upload in [(Equals(chroot.http_url), Equals(''), Equals(''))] +
235+ extra_uploads]
236
237 extra_args = {
238 'arch_indep': arch_indep,
239@@ -149,8 +177,9 @@
240 'trusted_keys': trusted_keys,
241 }
242 build_log = [
243- ('build', build.build_cookie, 'binarypackage',
244- chroot.content.sha1, filemap_names, extra_args)]
245+ MatchesListwise([Equals(arg) for arg in (
246+ 'build', build.build_cookie, 'binarypackage',
247+ chroot.content.sha1, filemap_names, extra_args)])]
248 result = upload_logs + build_log
249 defer.returnValue(result)
250
251@@ -244,13 +273,6 @@
252 sprf = build.source_package_release.addFile(
253 self.factory.makeLibraryFileAlias(db_only=True),
254 filetype=SourcePackageFileType.ORIG_TARBALL)
255- sprf_url = (
256- 'http://private-ppa.launchpad.dev/%s/%s/ubuntu/pool/%s/%s'
257- % (archive.owner.name, archive.name,
258- poolify(
259- build.source_package_release.sourcepackagename.name,
260- 'main'),
261- sprf.libraryfile.filename))
262 lf = self.factory.makeLibraryFileAlias()
263 transaction.commit()
264 build.distro_arch_series.addOrUpdateChroot(lf)
265@@ -262,7 +284,10 @@
266 interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
267 yield self.assertExpectedInteraction(
268 slave.call_log, builder, build, lf, archive, ArchivePurpose.PPA,
269- extra_uploads=[(sprf_url, 'buildd', 'sekrit')],
270+ extra_uploads=[(
271+ Equals(sprf.libraryfile.https_url), Equals(''),
272+ BinaryPackageBuildMacaroonVerifies(
273+ build, sprf.libraryfile.id))],
274 filemap_names=[sprf.libraryfile.filename])
275
276 @defer.inlineCallbacks