Launchpad itself

Merge ~cjwatson/launchpad:built-using-guard-copying into launchpad:master

  1. Git
  2. lp:~cjwatson/launchpad
  3. built-using-guard-copying
  4. Merge into master
Proposed by Colin Watson
Status: Needs review
Proposed branch: ~cjwatson/launchpad:built-using-guard-copying
Merge into: launchpad:master
Prerequisite: ~cjwatson/launchpad:built-using-guard-deletion
Diff against target: 311 lines (+226/-3)
5 files modified
lib/lp/soyuz/interfaces/binarysourcereference.py (+17/-0)
lib/lp/soyuz/model/binarysourcereference.py (+29/-1)
lib/lp/soyuz/scripts/packagecopier.py (+26/-2)
lib/lp/soyuz/scripts/tests/test_copypackage.py (+95/-0)
lib/lp/soyuz/tests/test_binarysourcereference.py (+59/-0)
Reviewer Review Type Date Requested Status
Ioana Lasc (community) Approve
Review via email: mp+382792@code.launchpad.net

Commit message

Guard copies of binaries with Built-Using references

Description of the change

If binaries have Built-Using references, then we need to make sure that we can resolve those references and keep the corresponding sources published while the binaries are published. Prevent copies of binaries if any such references can't be resolved in the target publishing context.

To post a comment you must log in.
Revision history for this message
Ioana Lasc (ilasc) wrote :

Looks good.

review: Approve
Reply

Unmerged commits

9c6dce7... by Colin Watson

Guard copies of binaries with Built-Using references

If binaries have Built-Using references, then we need to make sure that
we can resolve those references and keep the corresponding sources
published while the binaries are published. Prevent copies of binaries
if any such references can't be resolved in the target publishing
context.

LP: #1868558

06ccc3e... by Colin Watson

Simplify tests using createFromSourcePackageReleases

c0c4aa2... by Colin Watson

Expand deletion guard to other pockets

It now checks all pockets that could legitimately depend on the one from
which the publication is being deleted.

d7fbcfd... by Colin Watson

Guard removal of sources referenced by Built-Using

Prevent SourcePackagePublishingHistory.requestDeletion from deleting
source publications that have Built-Using references from active binary
publications in the same archive and suite.

This isn't necessarily complete: in particular, it can miss references
from other pockets, and in any case it might race with a build still in
progress. The intent of this is not to ensure integrity, but to avoid
some easily-detectable mistakes that could cause confusion.

LP: #1868558

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/lib/lp/soyuz/interfaces/binarysourcereference.py b/lib/lp/soyuz/interfaces/binarysourcereference.py
index e625cf9..f702959 100644
--- a/lib/lp/soyuz/interfaces/binarysourcereference.py
+++ b/lib/lp/soyuz/interfaces/binarysourcereference.py
@@ -99,3 +99,20 @@ class IBinarySourceReferenceSet(Interface):
99 pointing to any of this sequence of `ISourcePackageRelease`s.99 pointing to any of this sequence of `ISourcePackageRelease`s.
100 :return: A `ResultSet` of matching `IBinarySourceReference`s.100 :return: A `ResultSet` of matching `IBinarySourceReference`s.
101 """101 """
102
103 def findMissingSources(archive, distroseries, pockets, reference_type,
104 binary_package_releases=None):
105 """Find references to unpublished sources in a given context.
106
107 :param archive: An `IArchive` to search for source publications.
108 :param distroseries: An `IDistroSeries` to search for source
109 publications.
110 :param pockets: A sequence of `PackagePublishingPocket`s to search
111 for source publications.
112 :param reference_type: A `BinarySourceReferenceType` to search for.
113 :param binary_package_releases: Only return references from any of
114 this sequence of `IBinaryPackageRelease`s.
115 :return: A `ResultSet` of matching `IBinarySourceReference`s where
116 the `source_package_release` is not published in the given
117 publishing context.
118 """
diff --git a/lib/lp/soyuz/model/binarysourcereference.py b/lib/lp/soyuz/model/binarysourcereference.py
index ca3563b..c2b41e0 100644
--- a/lib/lp/soyuz/model/binarysourcereference.py
+++ b/lib/lp/soyuz/model/binarysourcereference.py
@@ -14,7 +14,9 @@ __all__ = [
14import warnings14import warnings
1515
16from debian.deb822 import PkgRelation16from debian.deb822 import PkgRelation
17from storm.expr import LeftJoin
17from storm.locals import (18from storm.locals import (
19 And,
18 Int,20 Int,
19 Reference,21 Reference,
20 )22 )
@@ -35,7 +37,10 @@ from lp.soyuz.interfaces.binarysourcereference import (
35 UnparsableBuiltUsing,37 UnparsableBuiltUsing,
36 )38 )
37from lp.soyuz.model.distroarchseries import DistroArchSeries39from lp.soyuz.model.distroarchseries import DistroArchSeries
38from lp.soyuz.model.publishing import BinaryPackagePublishingHistory40from lp.soyuz.model.publishing import (
41 BinaryPackagePublishingHistory,
42 SourcePackagePublishingHistory,
43 )
3944
4045
41@implementer(IBinarySourceReference)46@implementer(IBinarySourceReference)
@@ -175,3 +180,26 @@ class BinarySourceReferenceSet:
175 spr.id for spr in source_package_releases))180 spr.id for spr in source_package_releases))
176 return IStore(BinarySourceReference).find(181 return IStore(BinarySourceReference).find(
177 BinarySourceReference, *clauses).config(distinct=True)182 BinarySourceReference, *clauses).config(distinct=True)
183
184 @classmethod
185 def findMissingSources(cls, archive, distroseries, pockets, reference_type,
186 binary_package_releases):
187 """See `IBinarySourceReferenceSet`."""
188 origin = [
189 BinarySourceReference,
190 LeftJoin(
191 SourcePackagePublishingHistory,
192 And(
193 BinarySourceReference.source_package_release_id ==
194 SourcePackagePublishingHistory.sourcepackagereleaseID,
195 SourcePackagePublishingHistory.archive == archive,
196 SourcePackagePublishingHistory.distroseries ==
197 distroseries,
198 SourcePackagePublishingHistory.pocket.is_in(pockets))),
199 ]
200 return IStore(BinarySourceReference).using(*origin).find(
201 BinarySourceReference,
202 BinarySourceReference.binary_package_release_id.is_in(
203 bpr.id for bpr in binary_package_releases),
204 BinarySourceReference.reference_type == reference_type,
205 SourcePackagePublishingHistory.id == None).config(distinct=True)
diff --git a/lib/lp/soyuz/scripts/packagecopier.py b/lib/lp/soyuz/scripts/packagecopier.py
index 3891af5..03f4c77 100644
--- a/lib/lp/soyuz/scripts/packagecopier.py
+++ b/lib/lp/soyuz/scripts/packagecopier.py
@@ -1,4 +1,4 @@
1# Copyright 2009-2015 Canonical Ltd. This software is licensed under the1# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4"""Package copying utilities."""4"""Package copying utilities."""
@@ -19,10 +19,17 @@ from zope.component import getUtility
19from zope.security.proxy import removeSecurityProxy19from zope.security.proxy import removeSecurityProxy
2020
21from lp.services.database.bulk import load_related21from lp.services.database.bulk import load_related
22from lp.soyuz.adapters.archivedependencies import pocket_dependencies
22from lp.soyuz.adapters.overrides import SourceOverride23from lp.soyuz.adapters.overrides import SourceOverride
23from lp.soyuz.enums import SourcePackageFormat24from lp.soyuz.enums import (
25 BinarySourceReferenceType,
26 SourcePackageFormat,
27 )
24from lp.soyuz.interfaces.archive import CannotCopy28from lp.soyuz.interfaces.archive import CannotCopy
25from lp.soyuz.interfaces.binarypackagebuild import BuildSetStatus29from lp.soyuz.interfaces.binarypackagebuild import BuildSetStatus
30from lp.soyuz.interfaces.binarysourcereference import (
31 IBinarySourceReferenceSet,
32 )
26from lp.soyuz.interfaces.publishing import (33from lp.soyuz.interfaces.publishing import (
27 active_publishing_status,34 active_publishing_status,
28 IBinaryPackagePublishingHistory,35 IBinaryPackagePublishingHistory,
@@ -439,6 +446,7 @@ class CopyChecker:
439 built_binaries = source.getBuiltBinaries(want_files=True)446 built_binaries = source.getBuiltBinaries(want_files=True)
440 if len(built_binaries) == 0:447 if len(built_binaries) == 0:
441 raise CannotCopy("source has no binaries to be copied")448 raise CannotCopy("source has no binaries to be copied")
449
442 # Deny copies of binary publications containing files with450 # Deny copies of binary publications containing files with
443 # expiration date set. We only set such value for immediate451 # expiration date set. We only set such value for immediate
444 # expiration of old superseded binaries, so no point in452 # expiration of old superseded binaries, so no point in
@@ -449,6 +457,22 @@ class CopyChecker:
449 if binary_file.libraryfile.expires is not None:457 if binary_file.libraryfile.expires is not None:
450 raise CannotCopy('source has expired binaries')458 raise CannotCopy('source has expired binaries')
451459
460 # Deny copies of binary publications that contain Built-Using
461 # references to sources that do not exist in the target. The
462 # dominator will not be able to rectify the situation.
463 bsr_set = getUtility(IBinarySourceReferenceSet)
464 missing_sources = bsr_set.findMissingSources(
465 self.archive, series, pocket_dependencies[pocket],
466 BinarySourceReferenceType.BUILT_USING,
467 [binary_pub.binarypackagerelease
468 for binary_pub in built_binaries])
469 if not missing_sources.is_empty():
470 # XXX cjwatson 2020-04-19: It may also be useful to show the
471 # specific Built-Using references that don't exist.
472 raise CannotCopy(
473 'source has binaries with Built-Using references that do '
474 'not exist in the target')
475
452 # Check if there is already a source with the same name and version476 # Check if there is already a source with the same name and version
453 # published in the destination archive.477 # published in the destination archive.
454 self._checkArchiveConflicts(source, series)478 self._checkArchiveConflicts(source, series)
diff --git a/lib/lp/soyuz/scripts/tests/test_copypackage.py b/lib/lp/soyuz/scripts/tests/test_copypackage.py
index 388fcf7..a41cfb1 100644
--- a/lib/lp/soyuz/scripts/tests/test_copypackage.py
+++ b/lib/lp/soyuz/scripts/tests/test_copypackage.py
@@ -1046,6 +1046,101 @@ class CopyCheckerTestCase(TestCaseWithFactory):
1046 copied_from_archive=binary.archive),1046 copied_from_archive=binary.archive),
1047 ]))1047 ]))
10481048
1049 def test_checkCopy_cannot_copy_dangling_built_using_references(self):
1050 # checkCopy() raises CannotCopy if the copy includes binaries and
1051 # the binaries contain Built-Using references to sources that do not
1052 # exist in the target.
1053
1054 # Create testing sources and binaries.
1055 source = self.test_publisher.getPubSource()
1056 built_using_source = self.test_publisher.getPubSource(
1057 sourcename='built-using')
1058 built_using_relationship = '%s (= %s)' % (
1059 built_using_source.sourcepackagerelease.name,
1060 built_using_source.sourcepackagerelease.version)
1061 self.test_publisher.getPubBinaries(
1062 pub_source=source, built_using=built_using_relationship)
1063
1064 # Create a fresh PPA which will be the destination copy.
1065 archive = self.factory.makeArchive(
1066 distribution=self.test_publisher.ubuntutest,
1067 purpose=ArchivePurpose.PPA)
1068 series = source.distroseries
1069 pocket = source.pocket
1070
1071 # Now source-only copies are allowed.
1072 copy_checker = CopyChecker(archive, include_binaries=False)
1073 self.assertIsNone(
1074 copy_checker.checkCopy(
1075 source, series, pocket, check_permissions=False))
1076
1077 # Copies with binaries are denied.
1078 copy_checker = CopyChecker(archive, include_binaries=True)
1079 self.assertRaisesWithContent(
1080 CannotCopy,
1081 'source has binaries with Built-Using references that do not '
1082 'exist in the target',
1083 copy_checker.checkCopy,
1084 source, series, pocket, check_permissions=False)
1085
1086 def test_checkCopy_can_copy_resolvable_built_using_references(self):
1087 # checkCopy() allows copying binaries with Built-Using references to
1088 # sources that exist in the target, even if no longer published or
1089 # in a pocket that the target depends on.
1090
1091 # Create testing sources and binaries.
1092 source = self.test_publisher.getPubSource()
1093 published_source = self.test_publisher.getPubSource(
1094 sourcename='published')
1095 superseded_source = self.test_publisher.getPubSource(
1096 sourcename='superseded')
1097 release_pocket_source = self.test_publisher.getPubSource(
1098 sourcename='release-pocket')
1099 relationships = [
1100 '%s (= %s)' % (
1101 spph.sourcepackagerelease.name,
1102 spph.sourcepackagerelease.version)
1103 for spph in (
1104 published_source, superseded_source, release_pocket_source)]
1105 self.test_publisher.getPubBinaries(
1106 built_using=', '.join(relationships),
1107 status=PackagePublishingStatus.PUBLISHED, pub_source=source)
1108 target_series = self.factory.makeDistroSeries(
1109 distribution=source.distroseries.distribution)
1110 getUtility(ISourcePackageFormatSelectionSet).add(
1111 target_series, SourcePackageFormat.FORMAT_1_0)
1112 self.factory.makeSourcePackagePublishingHistory(
1113 distroseries=target_series, archive=source.archive,
1114 sourcepackagerelease=published_source.sourcepackagerelease,
1115 pocket=PackagePublishingPocket.PROPOSED,
1116 status=PackagePublishingStatus.PUBLISHED)
1117 self.factory.makeSourcePackagePublishingHistory(
1118 distroseries=target_series, archive=source.archive,
1119 sourcepackagerelease=superseded_source.sourcepackagerelease,
1120 pocket=PackagePublishingPocket.PROPOSED,
1121 status=PackagePublishingStatus.SUPERSEDED)
1122 self.factory.makeSourcePackagePublishingHistory(
1123 distroseries=target_series, archive=source.archive,
1124 sourcepackagerelease=release_pocket_source.sourcepackagerelease,
1125 pocket=PackagePublishingPocket.RELEASE,
1126 status=PackagePublishingStatus.PUBLISHED)
1127
1128 # Copies of binaries are permitted.
1129 copy_checker = CopyChecker(source.archive, include_binaries=True)
1130 copy_checker.checkCopy(
1131 source, target_series, PackagePublishingPocket.PROPOSED,
1132 check_permissions=False)
1133
1134 # Since some of the sources were only published in PROPOSED, copies
1135 # of binaries to RELEASE that refer to them are denied.
1136 self.assertRaisesWithContent(
1137 CannotCopy,
1138 'source has binaries with Built-Using references that do not '
1139 'exist in the target',
1140 copy_checker.checkCopy,
1141 source, target_series, PackagePublishingPocket.RELEASE,
1142 check_permissions=False)
1143
10491144
1050class BaseDoCopyTests:1145class BaseDoCopyTests:
10511146
diff --git a/lib/lp/soyuz/tests/test_binarysourcereference.py b/lib/lp/soyuz/tests/test_binarysourcereference.py
index 0c4db7f..b8ebcef 100644
--- a/lib/lp/soyuz/tests/test_binarysourcereference.py
+++ b/lib/lp/soyuz/tests/test_binarysourcereference.py
@@ -306,3 +306,62 @@ class TestBinarySourceReference(TestCaseWithFactory):
306 source_package_release=bsr.source_package_release,306 source_package_release=bsr.source_package_release,
307 reference_type=BinarySourceReferenceType.BUILT_USING)307 reference_type=BinarySourceReferenceType.BUILT_USING)
308 for bsr in [bsrs[0], bsrs[2]])))308 for bsr in [bsrs[0], bsrs[2]])))
309
310 def test_findMissingSources(self):
311 # findMissingSources finds references whose source publications
312 # aren't present in the given publishing context.
313 archive = self.factory.makeArchive()
314 distroseries = archive.distribution.currentseries
315 pockets = (
316 PackagePublishingPocket.RELEASE, PackagePublishingPocket.PROPOSED)
317 spphs = [
318 self.factory.makeSourcePackagePublishingHistory(
319 archive=archive, distroseries=distroseries, pocket=pocket)
320 for pocket in pockets]
321 bprs = []
322 for spph in spphs:
323 build = self.factory.makeBinaryPackageBuild(
324 distroarchseries=self.factory.makeDistroArchSeries(
325 distroseries=spph.distroseries),
326 archive=spph.archive, pocket=spph.pocket)
327 bprs.append(self.factory.makeBinaryPackageRelease(build=build))
328 for bpr, spph in zip(bprs, spphs):
329 self.reference_set.createFromSourcePackageReleases(
330 bpr, [spph.sourcepackagerelease],
331 BinarySourceReferenceType.BUILT_USING)
332 self.assertTrue(
333 self.reference_set.findMissingSources(
334 archive, distroseries, pockets,
335 BinarySourceReferenceType.BUILT_USING, bprs).is_empty())
336 # Try searching with slight mismatches; findMissingSources should
337 # return appropriate results since the necessary SPPHs aren't
338 # present.
339 self.assertThat(
340 self.reference_set.findMissingSources(
341 self.factory.makeArchive(), distroseries, pockets,
342 BinarySourceReferenceType.BUILT_USING, bprs),
343 MatchesSetwise(*(
344 MatchesStructure.byEquality(
345 binary_package_release=bpr,
346 source_package_release=spph.sourcepackagerelease,
347 reference_type=BinarySourceReferenceType.BUILT_USING)
348 for bpr, spph in zip(bprs, spphs))))
349 self.assertThat(
350 self.reference_set.findMissingSources(
351 archive, self.factory.makeDistroSeries(), pockets,
352 BinarySourceReferenceType.BUILT_USING, bprs),
353 MatchesSetwise(*(
354 MatchesStructure.byEquality(
355 binary_package_release=bpr,
356 source_package_release=spph.sourcepackagerelease,
357 reference_type=BinarySourceReferenceType.BUILT_USING)
358 for bpr, spph in zip(bprs, spphs))))
359 self.assertThat(
360 self.reference_set.findMissingSources(
361 archive, distroseries, [PackagePublishingPocket.PROPOSED],
362 BinarySourceReferenceType.BUILT_USING, bprs),
363 MatchesSetwise(
364 MatchesStructure.byEquality(
365 binary_package_release=bprs[0],
366 source_package_release=spphs[0].sourcepackagerelease,
367 reference_type=BinarySourceReferenceType.BUILT_USING)))

Subscribers

People subscribed via source and target branches

to status/vote changes: