Launchpad itself

Merge lp:~cjwatson/launchpad/build-private-bpb-immediately into lp:launchpad

  1. build-private-bpb-immediately
  2. Merge into devel
Proposed by Colin Watson
Status: Rejected
Rejected by: Colin Watson
Proposed branch: lp:~cjwatson/launchpad/build-private-bpb-immediately
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/librarian-accept-macaroon
Diff against target: 276 lines (+60/-64)
4 files modified
lib/lp/buildmaster/tests/test_builder.py (+4/-4)
lib/lp/soyuz/model/binarypackagebuild.py (+3/-27)
lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+12/-17)
lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+41/-16)
To merge this branch: bzr merge lp:~cjwatson/launchpad/build-private-bpb-immediately
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+345104@code.launchpad.net

Commit message

Dispatch private BPBs immediately, using macaroon auth for source files.

Description of the change

The prerequisite branch must be deployed to the librarian before landing this.

I haven't yet tested this beyond what's in the test suite.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) wrote :

Superseded by https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/373741.

Unmerged revisions

18632. By Colin Watson

Dispatch private BPBs immediately, using macaroon auth for source files.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
--- lib/lp/buildmaster/tests/test_builder.py 2018-01-10 10:45:24 +0000
+++ lib/lp/buildmaster/tests/test_builder.py 2018-05-04 17:00:50 +0000
@@ -401,13 +401,13 @@
401 build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)401 build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
402 self.assertEqual('joesppa', build.archive.name)402 self.assertEqual('joesppa', build.archive.name)
403403
404 # If the source for the build is still pending, it won't be404 # If the source for the build is still pending, it will still be
405 # dispatched because the builder has to fetch the source files405 # dispatched: the builder will use macaroon authentication to fetch
406 # from the (password protected) repo area, not the librarian.406 # the source files from the librarian.
407 pub = build.current_source_publication407 pub = build.current_source_publication
408 pub.status = PackagePublishingStatus.PENDING408 pub.status = PackagePublishingStatus.PENDING
409 candidate = removeSecurityProxy(self.builder4)._findBuildCandidate()409 candidate = removeSecurityProxy(self.builder4)._findBuildCandidate()
410 self.assertNotEqual(next_job.id, candidate.id)410 self.assertEqual(next_job.id, candidate.id)
411411
412412
413class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):413class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
414414
=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
--- lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
+++ lib/lp/soyuz/model/binarypackagebuild.py 2018-05-04 17:00:50 +0000
@@ -87,10 +87,7 @@
87 )87 )
88from lp.services.macaroons.interfaces import IMacaroonIssuer88from lp.services.macaroons.interfaces import IMacaroonIssuer
89from lp.soyuz.adapters.buildarch import determine_architectures_to_build89from lp.soyuz.adapters.buildarch import determine_architectures_to_build
90from lp.soyuz.enums import (90from lp.soyuz.enums import ArchivePurpose
91 ArchivePurpose,
92 PackagePublishingStatus,
93 )
94from lp.soyuz.interfaces.archive import (91from lp.soyuz.interfaces.archive import (
95 InvalidExternalDependencies,92 InvalidExternalDependencies,
96 validate_external_dependencies,93 validate_external_dependencies,
@@ -1210,33 +1207,12 @@
1210 @staticmethod1207 @staticmethod
1211 def addCandidateSelectionCriteria(processor, virtualized):1208 def addCandidateSelectionCriteria(processor, virtualized):
1212 """See `ISpecificBuildFarmJobSource`."""1209 """See `ISpecificBuildFarmJobSource`."""
1213 private_statuses = (
1214 PackagePublishingStatus.PUBLISHED,
1215 PackagePublishingStatus.SUPERSEDED,
1216 PackagePublishingStatus.DELETED,
1217 )
1218 return """1210 return """
1219 SELECT TRUE FROM Archive, BinaryPackageBuild, DistroArchSeries1211 SELECT TRUE FROM BinaryPackageBuild
1220 WHERE1212 WHERE
1221 BinaryPackageBuild.build_farm_job = BuildQueue.build_farm_job AND1213 BinaryPackageBuild.build_farm_job = BuildQueue.build_farm_job AND
1222 BinaryPackageBuild.distro_arch_series =
1223 DistroArchSeries.id AND
1224 BinaryPackageBuild.archive = Archive.id AND
1225 ((Archive.private IS TRUE AND
1226 EXISTS (
1227 SELECT SourcePackagePublishingHistory.id
1228 FROM SourcePackagePublishingHistory
1229 WHERE
1230 SourcePackagePublishingHistory.distroseries =
1231 DistroArchSeries.distroseries AND
1232 SourcePackagePublishingHistory.sourcepackagerelease =
1233 BinaryPackageBuild.source_package_release AND
1234 SourcePackagePublishingHistory.archive = Archive.id AND
1235 SourcePackagePublishingHistory.status IN %s))
1236 OR
1237 archive.private IS FALSE) AND
1238 BinaryPackageBuild.status = %s1214 BinaryPackageBuild.status = %s
1239 """ % sqlvalues(private_statuses, BuildStatus.NEEDSBUILD)1215 """ % sqlvalues(BuildStatus.NEEDSBUILD)
12401216
1241 @staticmethod1217 @staticmethod
1242 def postprocessCandidate(job, logger):1218 def postprocessCandidate(job, logger):
12431219
=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py'
--- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
@@ -1,4 +1,4 @@
1# Copyright 2009-2017 Canonical Ltd. This software is licensed under the1# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4"""Builder behaviour for binary package builds."""4"""Builder behaviour for binary package builds."""
@@ -10,7 +10,9 @@
10 ]10 ]
1111
12from twisted.internet import defer12from twisted.internet import defer
13from zope.component import getUtility
13from zope.interface import implementer14from zope.interface import implementer
15from zope.security.proxy import removeSecurityProxy
1416
15from lp.buildmaster.interfaces.builder import CannotBuild17from lp.buildmaster.interfaces.builder import CannotBuild
16from lp.buildmaster.interfaces.buildfarmjobbehaviour import (18from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
@@ -20,16 +22,13 @@
20 BuildFarmJobBehaviourBase,22 BuildFarmJobBehaviourBase,
21 )23 )
22from lp.registry.interfaces.pocket import PackagePublishingPocket24from lp.registry.interfaces.pocket import PackagePublishingPocket
23from lp.services.webapp import (25from lp.services.macaroons.interfaces import IMacaroonIssuer
24 canonical_url,26from lp.services.webapp import canonical_url
25 urlappend,
26 )
27from lp.soyuz.adapters.archivedependencies import (27from lp.soyuz.adapters.archivedependencies import (
28 get_primary_current_component,28 get_primary_current_component,
29 get_sources_list_for_building,29 get_sources_list_for_building,
30 )30 )
31from lp.soyuz.enums import ArchivePurpose31from lp.soyuz.enums import ArchivePurpose
32from lp.soyuz.model.publishing import makePoolPath
3332
3433
35@implementer(IBuildFarmJobBehaviour)34@implementer(IBuildFarmJobBehaviour)
@@ -63,14 +62,9 @@
63 # Build filemap structure with the files required in this build62 # Build filemap structure with the files required in this build
64 # and send them to the slave.63 # and send them to the slave.
65 if self.build.archive.private:64 if self.build.archive.private:
66 # Builds in private archive may have restricted files that65 issuer = getUtility(IMacaroonIssuer, 'binary-package-build')
67 # we can't obtain from the public librarian. Prepare a pool66 password = removeSecurityProxy(issuer).issueMacaroon(
68 # URL from which to fetch them.67 self.build).serialize()
69 pool_url = urlappend(
70 self.build.archive.archive_url,
71 makePoolPath(
72 self.build.source_package_release.sourcepackagename.name,
73 self.build.current_component.name))
74 filemap = {}68 filemap = {}
75 for source_file in self.build.source_package_release.files:69 for source_file in self.build.source_package_release.files:
76 lfa = source_file.libraryfile70 lfa = source_file.libraryfile
@@ -80,9 +74,10 @@
80 else:74 else:
81 filemap[lfa.filename] = {75 filemap[lfa.filename] = {
82 'sha1': lfa.content.sha1,76 'sha1': lfa.content.sha1,
83 'url': urlappend(pool_url, lfa.filename),77 'url': lfa.https_url,
84 'username': 'buildd',78 'username': '',
85 'password': self.build.archive.buildd_secret}79 'password': password,
80 }
86 return filemap81 return filemap
8782
88 @defer.inlineCallbacks83 @defer.inlineCallbacks
8984
=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-05-04 17:00:50 +0000
@@ -12,8 +12,15 @@
12import shutil12import shutil
13import tempfile13import tempfile
1414
15from pymacaroons import Macaroon
15from storm.store import Store16from storm.store import Store
16from testtools.matchers import MatchesListwise17from testtools.matchers import (
18 Contains,
19 Equals,
20 Matcher,
21 MatchesListwise,
22 Mismatch,
23 )
17from testtools.twistedsupport import AsynchronousDeferredRunTest24from testtools.twistedsupport import AsynchronousDeferredRunTest
18import transaction25import transaction
19from twisted.internet import defer26from twisted.internet import defer
@@ -21,7 +28,6 @@
21from zope.component import getUtility28from zope.component import getUtility
22from zope.security.proxy import removeSecurityProxy29from zope.security.proxy import removeSecurityProxy
2330
24from lp.archivepublisher.diskpool import poolify
25from lp.archivepublisher.interfaces.archivesigningkey import (31from lp.archivepublisher.interfaces.archivesigningkey import (
26 IArchiveSigningKey,32 IArchiveSigningKey,
27 )33 )
@@ -58,6 +64,7 @@
58from lp.services.config import config64from lp.services.config import config
59from lp.services.librarian.interfaces import ILibraryFileAliasSet65from lp.services.librarian.interfaces import ILibraryFileAliasSet
60from lp.services.log.logger import BufferLogger66from lp.services.log.logger import BufferLogger
67from lp.services.macaroons.interfaces import IMacaroonIssuer
61from lp.services.webapp import canonical_url68from lp.services.webapp import canonical_url
62from lp.soyuz.adapters.archivedependencies import (69from lp.soyuz.adapters.archivedependencies import (
63 get_sources_list_for_building,70 get_sources_list_for_building,
@@ -74,6 +81,26 @@
74from lp.testing.layers import LaunchpadZopelessLayer81from lp.testing.layers import LaunchpadZopelessLayer
7582
7683
84class BinaryPackageBuildMacaroonVerifies(Matcher):
85 """Matches if a binary-package-build macaroon passes verification."""
86
87 def __init__(self, build, lfa_id):
88 self.build = build
89 self.lfa_id = lfa_id
90
91 def match(self, macaroon_raw):
92 macaroon = Macaroon.deserialize(macaroon_raw)
93 expected_caveat = (
94 "lp.binary-package-build %s" % removeSecurityProxy(self.build).id)
95 mismatch = Contains(expected_caveat).match(
96 [caveat.caveat_id for caveat in macaroon.caveats])
97 if mismatch is not None:
98 return mismatch
99 issuer = getUtility(IMacaroonIssuer, "binary-package-build")
100 if not issuer.verifyMacaroon(macaroon, self.lfa_id):
101 return Mismatch("Macaroon does not verify")
102
103
77class TestBinaryBuildPackageBehaviour(TestCaseWithFactory):104class TestBinaryBuildPackageBehaviour(TestCaseWithFactory):
78 """Tests for the BinaryPackageBuildBehaviour.105 """Tests for the BinaryPackageBuildBehaviour.
79106
@@ -98,7 +125,7 @@
98 expected = yield self.makeExpectedInteraction(125 expected = yield self.makeExpectedInteraction(
99 builder, build, chroot, archive, archive_purpose, component,126 builder, build, chroot, archive, archive_purpose, component,
100 extra_uploads, filemap_names)127 extra_uploads, filemap_names)
101 self.assertEqual(expected, call_log)128 self.assertThat(call_log, MatchesListwise(expected))
102129
103 @defer.inlineCallbacks130 @defer.inlineCallbacks
104 def makeExpectedInteraction(self, builder, build, chroot, archive,131 def makeExpectedInteraction(self, builder, build, chroot, archive,
@@ -115,7 +142,7 @@
115 builder. We specify this separately from the archive because142 builder. We specify this separately from the archive because
116 sometimes the behaviour object has to give a different purpose143 sometimes the behaviour object has to give a different purpose
117 in order to trick the slave into building correctly.144 in order to trick the slave into building correctly.
118 :return: A list of the calls we expect to be made.145 :return: A list of matchers for the calls we expect to be made.
119 """146 """
120 das = build.distro_arch_series147 das = build.distro_arch_series
121 ds_name = das.distroseries.name148 ds_name = das.distroseries.name
@@ -131,8 +158,9 @@
131 extra_uploads = []158 extra_uploads = []
132159
133 upload_logs = [160 upload_logs = [
134 ('ensurepresent',) + upload161 MatchesListwise((Equals('ensurepresent'),) + upload)
135 for upload in [(chroot.http_url, '', '')] + extra_uploads]162 for upload in [(Equals(chroot.http_url), Equals(''), Equals(''))] +
163 extra_uploads]
136164
137 extra_args = {165 extra_args = {
138 'arch_indep': arch_indep,166 'arch_indep': arch_indep,
@@ -149,8 +177,9 @@
149 'trusted_keys': trusted_keys,177 'trusted_keys': trusted_keys,
150 }178 }
151 build_log = [179 build_log = [
152 ('build', build.build_cookie, 'binarypackage',180 MatchesListwise([Equals(arg) for arg in (
153 chroot.content.sha1, filemap_names, extra_args)]181 'build', build.build_cookie, 'binarypackage',
182 chroot.content.sha1, filemap_names, extra_args)])]
154 result = upload_logs + build_log183 result = upload_logs + build_log
155 defer.returnValue(result)184 defer.returnValue(result)
156185
@@ -244,13 +273,6 @@
244 sprf = build.source_package_release.addFile(273 sprf = build.source_package_release.addFile(
245 self.factory.makeLibraryFileAlias(db_only=True),274 self.factory.makeLibraryFileAlias(db_only=True),
246 filetype=SourcePackageFileType.ORIG_TARBALL)275 filetype=SourcePackageFileType.ORIG_TARBALL)
247 sprf_url = (
248 'http://private-ppa.launchpad.dev/%s/%s/ubuntu/pool/%s/%s'
249 % (archive.owner.name, archive.name,
250 poolify(
251 build.source_package_release.sourcepackagename.name,
252 'main'),
253 sprf.libraryfile.filename))
254 lf = self.factory.makeLibraryFileAlias()276 lf = self.factory.makeLibraryFileAlias()
255 transaction.commit()277 transaction.commit()
256 build.distro_arch_series.addOrUpdateChroot(lf)278 build.distro_arch_series.addOrUpdateChroot(lf)
@@ -262,7 +284,10 @@
262 interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())284 interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
263 yield self.assertExpectedInteraction(285 yield self.assertExpectedInteraction(
264 slave.call_log, builder, build, lf, archive, ArchivePurpose.PPA,286 slave.call_log, builder, build, lf, archive, ArchivePurpose.PPA,
265 extra_uploads=[(sprf_url, 'buildd', 'sekrit')],287 extra_uploads=[(
288 Equals(sprf.libraryfile.https_url), Equals(''),
289 BinaryPackageBuildMacaroonVerifies(
290 build, sprf.libraryfile.id))],
266 filemap_names=[sprf.libraryfile.filename])291 filemap_names=[sprf.libraryfile.filename])
267292
268 @defer.inlineCallbacks293 @defer.inlineCallbacks