Launchpad itself

Merge ~pelpsi/launchpad:unembargo-bug-build-info-missing into launchpad:master

  1. Git
  2. lp:~pelpsi/launchpad
  3. unembargo-bug-build-info-missing
  4. Merge into master
Proposed by Simone Pelosi
Status: Work in progress
Proposed branch: ~pelpsi/launchpad:unembargo-bug-build-info-missing
Merge into: launchpad:master
Diff against target: 498 lines (+352/-11)
7 files modified
lib/lp/soyuz/browser/tests/test_publishing_webservice.py (+109/-6)
lib/lp/soyuz/interfaces/publishing.py (+14/-0)
lib/lp/soyuz/model/packagecopyjob.py (+5/-2)
lib/lp/soyuz/model/publishing.py (+21/-0)
lib/lp/soyuz/scripts/packagecopier.py (+3/-1)
lib/lp/soyuz/tests/test_packagecopyjob.py (+108/-0)
lib/lp/soyuz/tests/test_publishing_models.py (+92/-2)
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+464705@code.launchpad.net

Commit message

Unembargo build info

Added buildinfo to unembargoed files.
Added custom upload files to unembargoed files.
Added a new endpoint buildMetadataFileUrls to retrieve
changes file, build_info and log without accessing directly the build.

LP: #2052796

To post a comment you must log in.

Unmerged commits

3401a5e... by Simone Pelosi

Unembargo build info

Added buildinfo to unembargoed files.
Added custom upload files to unembargoed files.
Added a new endpoint buildMetadataFileUrls to retrieve
changes file, build_info and log without accessing directly the build.

LP: #2052796

Succeeded
[SUCCEEDED] docs:0 (build)
[SUCCEEDED] lint:0 (build)
[SUCCEEDED] mypy:0 (build)
1 → 3 of 3 results First • Previous • Next • Last

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/soyuz/browser/tests/test_publishing_webservice.py b/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
2index 2aee350..b167e61 100644
3--- a/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
4+++ b/lib/lp/soyuz/browser/tests/test_publishing_webservice.py
5@@ -15,9 +15,14 @@ from lp.services.librarian.browser import ProxiedLibraryFileAlias
6 from lp.services.webapp.interfaces import OAuthPermission
7 from lp.soyuz.adapters.proxiedsourcefiles import ProxiedSourceLibraryFileAlias
8 from lp.soyuz.enums import BinaryPackageFormat
9-from lp.soyuz.interfaces.publishing import IPublishingSet
10+from lp.soyuz.interfaces.publishing import (
11+ IPublishingSet,
12+ PackagePublishingStatus,
13+)
14+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
15 from lp.testing import (
16 TestCaseWithFactory,
17+ admin_logged_in,
18 api_url,
19 login_person,
20 person_logged_in,
21@@ -167,14 +172,47 @@ class SourcePackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
22 class BinaryPackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
23 layer = LaunchpadFunctionalLayer
24
25+ def setUp(self):
26+ super().setUp()
27+ self.processor = self.factory.makeProcessor(supports_virtualized=True)
28+ self.distroseries = self.factory.makeDistroSeries()
29+ self.das = self.factory.makeDistroArchSeries(
30+ distroseries=self.distroseries, processor=self.processor
31+ )
32+ with admin_logged_in():
33+ self.publisher = SoyuzTestPublisher()
34+ self.publisher.prepareBreezyAutotest()
35+ self.distroseries.nominatedarchindep = self.das
36+ self.publisher.addFakeChroots(distroseries=self.distroseries)
37+ self.builder = self.factory.makeBuilder(
38+ processors=[self.processor]
39+ )
40+
41 def make_bpph_for(self, person):
42+ spn = self.factory.getUniqueString()
43+ version = "%s.1" % self.factory.getUniqueInteger()
44 with person_logged_in(person):
45- bpr = self.factory.makeBinaryPackageRelease()
46- self.factory.makeBinaryPackageFile(binarypackagerelease=bpr)
47- bpph = self.factory.makeBinaryPackagePublishingHistory(
48- binarypackagerelease=bpr
49+ spph = self.publisher.getPubSource(
50+ sourcename=spn,
51+ version=version,
52+ distroseries=self.distroseries,
53+ status=PackagePublishingStatus.PUBLISHED,
54+ )
55+ with admin_logged_in():
56+ binary = self.publisher.getPubBinaries(
57+ binaryname=spn,
58+ distroseries=self.distroseries,
59+ pub_source=spph,
60+ version=version,
61+ builder=self.builder,
62+ )
63+ binary[0].build.addBuildInfo(
64+ self.factory.makeLibraryFileAlias(
65+ filename="build_info.info", restricted=True
66+ )
67 )
68- return bpph, api_url(bpph)
69+
70+ return binary[0], api_url(binary[0])
71
72 def test_binaryFileUrls(self):
73 person = self.factory.makePerson()
74@@ -239,6 +277,71 @@ class BinaryPackagePublishingHistoryWebserviceTests(TestCaseWithFactory):
75 ]
76 self.assertContentEqual(expected_info, info)
77
78+ def test_buildMetadataFileUrls(self):
79+ person = self.factory.makePerson()
80+ webservice = webservice_for_person(
81+ person, permission=OAuthPermission.READ_PUBLIC
82+ )
83+ bpph, url = self.make_bpph_for(person)
84+
85+ response = webservice.named_get(
86+ url, "buildMetadataFileUrls", api_version="devel"
87+ )
88+
89+ self.assertEqual(200, response.status)
90+ urls = response.jsonBody()
91+
92+ with person_logged_in(person):
93+ upload_changesfile = (
94+ bpph.binarypackagerelease.build.upload_changesfile
95+ )
96+ log = bpph.binarypackagerelease.build.log
97+ build_info = bpph.binarypackagerelease.build.buildinfo
98+
99+ expected_urls = [
100+ ProxiedLibraryFileAlias(
101+ upload_changesfile, bpph.archive
102+ ).http_url,
103+ ProxiedLibraryFileAlias(log, bpph.archive).http_url,
104+ ProxiedLibraryFileAlias(build_info, bpph.archive).http_url,
105+ ]
106+ self.assertEqual(expected_urls, urls)
107+
108+ def test_buildMetadataFileUrls_include_meta(self):
109+ person = self.factory.makePerson()
110+ webservice = webservice_for_person(
111+ person, permission=OAuthPermission.READ_PUBLIC
112+ )
113+ bpph, url = self.make_bpph_for(person)
114+
115+ response = webservice.named_get(
116+ url,
117+ "buildMetadataFileUrls",
118+ include_meta=True,
119+ api_version="devel",
120+ )
121+
122+ self.assertEqual(200, response.status)
123+ info = response.jsonBody()
124+ with person_logged_in(person):
125+ files = []
126+ files.append(bpph.binarypackagerelease.build.upload_changesfile)
127+ files.append(bpph.binarypackagerelease.build.log)
128+ files.append(bpph.binarypackagerelease.build.buildinfo)
129+
130+ expected_info = [
131+ {
132+ "url": ProxiedLibraryFileAlias(
133+ file, bpph.archive
134+ ).http_url,
135+ "size": file.content.filesize,
136+ "sha1": file.content.sha1,
137+ "sha256": file.content.sha256,
138+ }
139+ for file in files
140+ ]
141+ self.assertEqual(expected_info, info)
142+
143 def test_ci_build(self):
144 person = self.factory.makePerson()
145 webservice = webservice_for_person(
146diff --git a/lib/lp/soyuz/interfaces/publishing.py b/lib/lp/soyuz/interfaces/publishing.py
147index a481aa3..c426ec7 100644
148--- a/lib/lp/soyuz/interfaces/publishing.py
149+++ b/lib/lp/soyuz/interfaces/publishing.py
150@@ -1041,6 +1041,20 @@ class IBinaryPackagePublishingHistoryPublic(IPublishingView):
151 :return: A collection of URLs for this binary.
152 """
153
154+ @export_read_operation()
155+ @operation_parameters(
156+ include_meta=Bool(title=_("Include Metadata"), required=False)
157+ )
158+ @operation_for_version("devel")
159+ def buildMetadataFileUrls(include_meta=False):
160+ """URLs for this build metadata files.
161+ The result includes changes file, build_info and log.
162+
163+ :param include_meta: Return a list of dicts with keys url, size,
164+ sha1, and sha256 for each URL instead of a simple list.
165+ :return: A collection of URLs for these metadata files.
166+ """
167+
168
169 class IBinaryPackagePublishingHistoryEdit(IPublishingEdit):
170 """A writeable binary package publishing record."""
171diff --git a/lib/lp/soyuz/model/packagecopyjob.py b/lib/lp/soyuz/model/packagecopyjob.py
172index 2776af2..d30f3af 100644
173--- a/lib/lp/soyuz/model/packagecopyjob.py
174+++ b/lib/lp/soyuz/model/packagecopyjob.py
175@@ -62,7 +62,7 @@ from lp.soyuz.interfaces.packagecopyjob import (
176 )
177 from lp.soyuz.interfaces.packagediff import PackageDiffAlreadyRequested
178 from lp.soyuz.interfaces.publishing import ISourcePackagePublishingHistory
179-from lp.soyuz.interfaces.queue import IPackageUploadSet
180+from lp.soyuz.interfaces.queue import IPackageUploadCustom, IPackageUploadSet
181 from lp.soyuz.interfaces.section import ISectionSet
182 from lp.soyuz.model.archive import Archive
183 from lp.soyuz.scripts.packagecopier import do_copy
184@@ -854,7 +854,10 @@ class PlainPackageCopyJob(PackageCopyJobDerived):
185 "Packages copied to %s:" % self.target_archive.displayname
186 )
187 for copy in copied_publications:
188- self.logger.debug(copy.displayname)
189+ if IPackageUploadCustom.providedBy(copy):
190+ self.logger.debug(copy.libraryfilealias.filename)
191+ else:
192+ self.logger.debug(copy.displayname)
193
194 def abort(self):
195 """Abort work."""
196diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py
197index 78749c0..44543e0 100644
198--- a/lib/lp/soyuz/model/publishing.py
199+++ b/lib/lp/soyuz/model/publishing.py
200@@ -1384,6 +1384,27 @@ class BinaryPackagePublishingHistory(StormBase, ArchivePublisherBase):
201 ]
202 return binary_urls
203
204+ def buildMetadataFileUrls(self, include_meta=False):
205+ """See `IBinaryPackagePublishingHistory`."""
206+ files = []
207+ if self.build.upload_changesfile:
208+ files.append(self.build.upload_changesfile)
209+ if self.build.log:
210+ files.append(self.build.log)
211+ if self.build.buildinfo:
212+ files.append(self.build.buildinfo)
213+ file_urls = proxied_urls([file for file in files], self.archive)
214+ if include_meta:
215+ meta = [
216+ (file.content.filesize, file.content.sha1, file.content.sha256)
217+ for file in files
218+ ]
219+ return [
220+ dict(url=url, size=size, sha1=sha1, sha256=sha256)
221+ for url, (size, sha1, sha256) in zip(file_urls, meta)
222+ ]
223+ return file_urls
224+
225
226 def expand_binary_requests(distroseries, binaries):
227 """Architecture-expand a dict of binary publication requests.
228diff --git a/lib/lp/soyuz/scripts/packagecopier.py b/lib/lp/soyuz/scripts/packagecopier.py
229index f377107..3e28161 100644
230--- a/lib/lp/soyuz/scripts/packagecopier.py
231+++ b/lib/lp/soyuz/scripts/packagecopier.py
232@@ -87,6 +87,8 @@ def update_files_privacy(pub_record):
233 package_files.append((package_upload, "changesfile"))
234 # Unrestrict the buildlog file as necessary.
235 package_files.append((build, "log"))
236+ # Unrestrict the buildinfo file as necessary.
237+ package_files.append((build, "buildinfo"))
238 elif IPackageUploadCustom.providedBy(pub_record):
239 # Unrestrict the custom files included
240 package_files.append((pub_record, "libraryfilealias"))
241@@ -911,7 +913,7 @@ def _do_direct_copy(
242 )
243 for custom in custom_files:
244 if custom_copier.isCopyable(custom):
245- custom_copier.copyUpload(custom)
246+ copies.append(custom_copier.copyUpload(custom))
247
248 # Always ensure the needed builds exist in the copy destination
249 # after copying the binaries.
250diff --git a/lib/lp/soyuz/tests/test_packagecopyjob.py b/lib/lp/soyuz/tests/test_packagecopyjob.py
251index 26fdd08..41e9c74 100644
252--- a/lib/lp/soyuz/tests/test_packagecopyjob.py
253+++ b/lib/lp/soyuz/tests/test_packagecopyjob.py
254@@ -1733,6 +1733,114 @@ class PlainPackageCopyJobTests(TestCaseWithFactory, LocalTestHelper):
255 self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
256 self.assertEqual(1, webhook.deliveries.count())
257
258+ def test_copying_unembargoes_build_files(self):
259+ # Inaccessible private builds aren't linked in builders'
260+ # current_build fields.
261+ # Copyable custom upload files are queued for republication when
262+ # they are copied.
263+ # Custom upload files should be unembargoed as well.
264+
265+ owner = self.factory.makePerson()
266+
267+ source_archive = self.factory.makeArchive(
268+ name="sourcearchive",
269+ owner=owner,
270+ distribution=self.distroseries.distribution,
271+ private=True,
272+ )
273+ target_archive = self.factory.makeArchive(
274+ name="targetarchive",
275+ owner=owner,
276+ distribution=self.distroseries.distribution,
277+ purpose=ArchivePurpose.PRIMARY,
278+ )
279+
280+ self.distroseries.status = SeriesStatus.CURRENT
281+ spph = self.publisher.getPubSource(
282+ pocket=PackagePublishingPocket.SECURITY, archive=source_archive
283+ )
284+ self.publisher.getPubBinaries(
285+ pocket=PackagePublishingPocket.SECURITY,
286+ pub_source=spph,
287+ archive=source_archive,
288+ )
289+ [build] = spph.getBuilds()
290+ build.addBuildInfo(
291+ self.factory.makeLibraryFileAlias(
292+ filename="build_info.info", restricted=True
293+ )
294+ )
295+ build.package_upload.addCustom(
296+ self.factory.makeLibraryFileAlias(
297+ filename="test.deb", restricted=True
298+ ),
299+ PackageUploadCustomFormat.DEBIAN_INSTALLER,
300+ )
301+ # Make the new librarian file available.
302+ self.layer.txn.commit()
303+ # Create the copy job.
304+ with person_logged_in(owner):
305+ target_archive.newPocketUploader(
306+ owner, PackagePublishingPocket.SECURITY
307+ )
308+ job = self.createCopyJobForSPPH(
309+ spph,
310+ source_archive,
311+ target_archive,
312+ requester=owner,
313+ target_pocket=PackagePublishingPocket.SECURITY,
314+ include_binaries=True,
315+ unembargo=True,
316+ )
317+
318+ # Start, accept, and run the job.
319+ self.runJob(job)
320+ self.assertEqual(JobStatus.SUSPENDED, job.status)
321+ switch_dbuser("launchpad_main")
322+ pu = (
323+ getUtility(IPackageUploadSet)
324+ .getByPackageCopyJobIDs([removeSecurityProxy(job).context.id])
325+ .one()
326+ )
327+ pu.acceptFromQueue()
328+ self.assertEqual(PackageUploadStatus.ACCEPTED, pu.status)
329+ self.runJob(job)
330+ self.assertEqual(PackageUploadStatus.DONE, pu.status)
331+
332+ uploads = list(
333+ self.distroseries.getPackageUploads(
334+ status=PackageUploadStatus.ACCEPTED,
335+ archive=target_archive,
336+ pocket=PackagePublishingPocket.SECURITY,
337+ )
338+ )
339+
340+ self.assertEqual(1, len(uploads))
341+ upload = uploads[0]
342+ self.assertEqual(
343+ PackageUploadCustomFormat.DEBIAN_INSTALLER,
344+ upload.customfiles[0].customformat,
345+ )
346+ # The upload is targeted to the right publishing context.
347+ self.assertEqual(target_archive, upload.archive)
348+ self.assertEqual(self.distroseries, upload.distroseries)
349+ self.assertEqual(PackagePublishingPocket.SECURITY, upload.pocket)
350+
351+ self.assertEqual(
352+ "test.deb", upload.customfiles[0].libraryfilealias.filename
353+ )
354+ self.assertEqual(
355+ False, upload.customfiles[0].libraryfilealias.restricted
356+ )
357+ copied_binaries = target_archive.getAllPublishedBinaries()
358+ for copied_binary in copied_binaries:
359+ for binary_file in copied_binary.binarypackagerelease.files:
360+ self.assertFalse(binary_file.libraryfile.restricted)
361+ copied_build = copied_binary.binarypackagerelease.build
362+ self.assertFalse(copied_build.log.restricted)
363+ self.assertFalse(copied_build.buildinfo.restricted)
364+ self.assertFalse(copied_build.upload_changesfile.restricted)
365+
366 def test_copying_unembargoes_files(self):
367 # The unembargo flag causes the job to unrestrict files.
368 self.distroseries.status = SeriesStatus.CURRENT
369diff --git a/lib/lp/soyuz/tests/test_publishing_models.py b/lib/lp/soyuz/tests/test_publishing_models.py
370index 755d797..e529d52 100644
371--- a/lib/lp/soyuz/tests/test_publishing_models.py
372+++ b/lib/lp/soyuz/tests/test_publishing_models.py
373@@ -21,7 +21,8 @@ from lp.soyuz.interfaces.publishing import (
374 PackagePublishingStatus,
375 )
376 from lp.soyuz.tests.test_binarypackagebuild import BaseTestCaseWithThreeBuilds
377-from lp.testing import TestCaseWithFactory, person_logged_in
378+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
379+from lp.testing import TestCaseWithFactory, admin_logged_in, person_logged_in
380 from lp.testing.layers import LaunchpadFunctionalLayer, LaunchpadZopelessLayer
381
382
383@@ -242,6 +243,22 @@ class TestSourcePackagePublishingHistory(TestCaseWithFactory):
384 class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
385 layer = LaunchpadFunctionalLayer
386
387+ def setUp(self):
388+ super().setUp()
389+ self.processor = self.factory.makeProcessor(supports_virtualized=True)
390+ self.distroseries = self.factory.makeDistroSeries()
391+ self.das = self.factory.makeDistroArchSeries(
392+ distroseries=self.distroseries, processor=self.processor
393+ )
394+ with admin_logged_in():
395+ self.publisher = SoyuzTestPublisher()
396+ self.publisher.prepareBreezyAutotest()
397+ self.distroseries.nominatedarchindep = self.das
398+ self.publisher.addFakeChroots(distroseries=self.distroseries)
399+ self.builder = self.factory.makeBuilder(
400+ processors=[self.processor]
401+ )
402+
403 def getURLsForBPPH(self, bpph, include_meta=False):
404 bpr = bpph.binarypackagerelease
405 archive = bpph.archive
406@@ -278,11 +295,35 @@ class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
407 binarypackagerelease=bpr, archive=archive
408 )
409
410+ def makeBPPHWithMetadata(self):
411+ spn = self.factory.getUniqueString()
412+ version = "%s.1" % self.factory.getUniqueInteger()
413+ with admin_logged_in():
414+ spph = self.publisher.getPubSource(
415+ sourcename=spn,
416+ version=version,
417+ distroseries=self.distroseries,
418+ status=PackagePublishingStatus.PUBLISHED,
419+ )
420+ binary = self.publisher.getPubBinaries(
421+ binaryname=spn,
422+ distroseries=self.distroseries,
423+ pub_source=spph,
424+ version=version,
425+ builder=self.builder,
426+ )
427+ binary[0].build.addBuildInfo(
428+ self.factory.makeLibraryFileAlias(
429+ filename="build_info.info", restricted=True
430+ )
431+ )
432+
433+ return binary[0]
434+
435 def test_binaryFileUrls_no_binaries(self):
436 bpph = self.makeBPPH(num_binaries=0)
437
438 urls = bpph.binaryFileUrls()
439-
440 self.assertContentEqual([], urls)
441
442 def test_binaryFileUrls_one_binary(self):
443@@ -324,6 +365,55 @@ class TestBinaryPackagePublishingHistory(TestCaseWithFactory):
444 expected_urls_meta, bpph.binaryFileUrls(include_meta=True)
445 )
446
447+ def test_buildMetadataFileUrls(self):
448+ bpph = self.makeBPPHWithMetadata()
449+
450+ urls = bpph.buildMetadataFileUrls()
451+ with admin_logged_in():
452+ upload_changesfile = (
453+ bpph.binarypackagerelease.build.upload_changesfile
454+ )
455+ log = bpph.binarypackagerelease.build.log
456+ build_info = bpph.binarypackagerelease.build.buildinfo
457+
458+ expected_urls = [
459+ ProxiedLibraryFileAlias(
460+ upload_changesfile, bpph.archive
461+ ).http_url,
462+ ProxiedLibraryFileAlias(log, bpph.archive).http_url,
463+ ProxiedLibraryFileAlias(build_info, bpph.archive).http_url,
464+ ]
465+ self.assertContentEqual(expected_urls, urls)
466+
467+ def test_buildMetadataFileUrls_include_meta(self):
468+ bpph = self.makeBPPHWithMetadata()
469+
470+ urls = bpph.buildMetadataFileUrls(True)
471+ with admin_logged_in():
472+ files = []
473+ files.append(bpph.binarypackagerelease.build.upload_changesfile)
474+ files.append(bpph.binarypackagerelease.build.log)
475+ files.append(bpph.binarypackagerelease.build.buildinfo)
476+
477+ expected_urls = [
478+ {
479+ "url": ProxiedLibraryFileAlias(
480+ file, bpph.archive
481+ ).http_url,
482+ "size": file.content.filesize,
483+ "sha1": file.content.sha1,
484+ "sha256": file.content.sha256,
485+ }
486+ for file in files
487+ ]
488+ self.assertContentEqual(expected_urls, urls)
489+
490+ def test_buildMetadataFileUrls_no_files(self):
491+ bpph = self.makeBPPH(num_binaries=0)
492+
493+ urls = bpph.buildMetadataFileUrls()
494+ self.assertContentEqual([], urls)
495+
496 def test_is_debug_false_for_deb(self):
497 bpph = self.factory.makeBinaryPackagePublishingHistory(
498 binpackageformat=BinaryPackageFormat.DEB

Subscribers

People subscribed via source and target branches

to status/vote changes: