Launchpad itself

Merge lp:~wgrant/launchpad/gpgkey-fks into lp:launchpad

  1. gpgkey-fks
  2. Merge into devel
Proposed by William Grant
Status: Merged
Merged at revision: 17939
Proposed branch: lp:~wgrant/launchpad/gpgkey-fks
Merge into: lp:launchpad
Prerequisite: lp:~wgrant/launchpad/gpgkey-fks-db
Diff against target: 464 lines (+199/-15)
11 files modified
database/schema/security.cfg (+4/-1)
lib/lp/archivepublisher/archivesigningkey.py (+8/-2)
lib/lp/registry/model/codeofconduct.py (+6/-2)
lib/lp/registry/model/distroseries.py (+10/-3)
lib/lp/scripts/garbo.py (+78/-0)
lib/lp/scripts/tests/test_garbo.py (+66/-1)
lib/lp/soyuz/configure.zcml (+6/-1)
lib/lp/soyuz/model/archive.py (+12/-5)
lib/lp/soyuz/model/queue.py (+3/-0)
lib/lp/soyuz/model/sourcepackagerelease.py (+4/-0)
lib/lp/soyuz/scripts/gina/handlers.py (+2/-0)
To merge this branch: bzr merge lp:~wgrant/launchpad/gpgkey-fks
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+287710@code.launchpad.net

Commit message

Set GPGKey FK replacement columns and add garbo jobs to backfill.

Description of the change

Set GPGKey FK replacement columns and add garbo jobs to backfill.

Using a Storm validator would be lovely, but construction gets angry when an incomplete object is flushed.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'database/schema/security.cfg'
2--- database/schema/security.cfg 2016-03-02 21:21:26 +0000
3+++ database/schema/security.cfg 2016-03-03 18:43:25 +0000
4@@ -2277,6 +2277,7 @@
5 public.accesspolicygrant = SELECT, DELETE
6 public.account = SELECT, DELETE
7 public.answercontact = SELECT, DELETE
8+public.archive = SELECT, UPDATE
9 public.branch = SELECT, UPDATE
10 public.branchjob = SELECT, DELETE
11 public.binarypackagename = SELECT
12@@ -2318,6 +2319,7 @@
13 public.milestonetag = SELECT
14 public.openidconsumerassociation = SELECT, DELETE
15 public.openidconsumernonce = SELECT, DELETE
16+public.packageupload = SELECT, UPDATE
17 public.person = SELECT, DELETE
18 public.personsettings = SELECT, UPDATE
19 public.product = SELECT, UPDATE
20@@ -2327,9 +2329,10 @@
21 public.previewdiff = SELECT, DELETE
22 public.revisionauthor = SELECT, UPDATE
23 public.revisioncache = SELECT, DELETE
24+public.signedcodeofconduct = SELECT, UPDATE
25 public.snapfile = SELECT, DELETE
26 public.sourcepackagename = SELECT
27-public.sourcepackagerelease = SELECT
28+public.sourcepackagerelease = SELECT, UPDATE
29 public.sourcepackagepublishinghistory = SELECT, UPDATE
30 public.suggestivepotemplate = INSERT, DELETE
31 public.teammembership = SELECT, DELETE
32
33=== modified file 'lib/lp/archivepublisher/archivesigningkey.py'
34--- lib/lp/archivepublisher/archivesigningkey.py 2015-09-13 18:30:51 +0000
35+++ lib/lp/archivepublisher/archivesigningkey.py 2016-03-03 18:43:25 +0000
36@@ -70,7 +70,10 @@
37 if self.archive != default_ppa:
38 if default_ppa.signing_key is None:
39 IArchiveSigningKey(default_ppa).generateSigningKey()
40- self.archive.signing_key = default_ppa.signing_key
41+ key = default_ppa.signing_key
42+ self.archive.signing_key = key
43+ self.archive.signing_key_owner = key.owner
44+ self.archive._signing_key_fingerprint = key.fingerprint
45 return
46
47 key_displayname = (
48@@ -105,9 +108,12 @@
49
50 algorithm = GPGKeyAlgorithm.items[pub_key.algorithm]
51 key_owner = getUtility(ILaunchpadCelebrities).ppa_key_guard
52- self.archive.signing_key = getUtility(IGPGKeySet).new(
53+ key = getUtility(IGPGKeySet).new(
54 key_owner, pub_key.keyid, pub_key.fingerprint, pub_key.keysize,
55 algorithm, active=True, can_encrypt=pub_key.can_encrypt)
56+ self.archive.signing_key = key
57+ self.archive.signing_key_owner = key.owner
58+ self.archive._signing_key_fingerprint = key.fingerprint
59
60 def signRepository(self, suite):
61 """See `IArchiveSigningKey`."""
62
63=== modified file 'lib/lp/registry/model/codeofconduct.py'
64--- lib/lp/registry/model/codeofconduct.py 2015-07-10 15:31:28 +0000
65+++ lib/lp/registry/model/codeofconduct.py 2016-03-03 18:43:25 +0000
66@@ -19,6 +19,7 @@
67 ForeignKey,
68 StringCol,
69 )
70+from storm.properties import Unicode
71 from zope.component import getUtility
72 from zope.interface import implementer
73
74@@ -181,6 +182,7 @@
75
76 signingkey = ForeignKey(foreignKey="GPGKey", dbName="signingkey",
77 notNull=False, default=None)
78+ signing_key_fingerprint = Unicode()
79
80 datecreated = UtcDateTimeCol(dbName='datecreated', notNull=True,
81 default=UTC_NOW)
82@@ -305,8 +307,10 @@
83 'space differences are acceptable).')
84
85 # Store the signature
86- signed = SignedCodeOfConduct(owner=user, signingkey=gpg,
87- signedcode=signedcode, active=True)
88+ signed = SignedCodeOfConduct(
89+ owner=user, signingkey=gpg,
90+ signing_key_fingerprint=gpg.fingerprint if gpg else None,
91+ signedcode=signedcode, active=True)
92
93 # Send Advertisement Email
94 subject = 'Your Code of Conduct signature has been acknowledged'
95
96=== modified file 'lib/lp/registry/model/distroseries.py'
97--- lib/lp/registry/model/distroseries.py 2016-02-05 20:28:29 +0000
98+++ lib/lp/registry/model/distroseries.py 2016-03-03 18:43:25 +0000
99@@ -1164,8 +1164,11 @@
100 architecturehintlist=architecturehintlist, component=component,
101 creator=creator, urgency=urgency, changelog=changelog,
102 changelog_entry=changelog_entry, dsc=dsc,
103- dscsigningkey=dscsigningkey, section=section, copyright=copyright,
104- upload_archive=archive,
105+ dscsigningkey=dscsigningkey,
106+ signing_key_owner=dscsigningkey.owner if dscsigningkey else None,
107+ signing_key_fingerprint=(
108+ dscsigningkey.fingerprint if dscsigningkey else None),
109+ section=section, copyright=copyright, upload_archive=archive,
110 dsc_maintainer_rfc822=dsc_maintainer_rfc822,
111 dsc_standards_version=dsc_standards_version,
112 dsc_format=dsc_format, dsc_binaries=dsc_binaries,
113@@ -1350,7 +1353,11 @@
114 return PackageUpload(
115 distroseries=self, status=PackageUploadStatus.NEW,
116 pocket=pocket, archive=archive, changesfile=changes_file_alias,
117- signing_key=signing_key, package_copy_job=package_copy_job)
118+ signing_key=signing_key,
119+ signing_key_owner=signing_key.owner if signing_key else None,
120+ signing_key_fingerprint=(
121+ signing_key.fingerprint if signing_key else None),
122+ package_copy_job=package_copy_job)
123
124 def getPackageUploadQueue(self, state):
125 """See `IDistroSeries`."""
126
127=== modified file 'lib/lp/scripts/garbo.py'
128--- lib/lp/scripts/garbo.py 2016-02-05 17:18:57 +0000
129+++ lib/lp/scripts/garbo.py 2016-03-03 18:43:25 +0000
130@@ -69,6 +69,7 @@
131 RevisionCache,
132 )
133 from lp.hardwaredb.model.hwdb import HWSubmission
134+from lp.registry.model.codeofconduct import SignedCodeOfConduct
135 from lp.registry.model.commercialsubscription import CommercialSubscription
136 from lp.registry.model.person import (
137 Person,
138@@ -124,6 +125,7 @@
139 from lp.soyuz.model.archive import Archive
140 from lp.soyuz.model.livefsbuild import LiveFSFile
141 from lp.soyuz.model.publishing import SourcePackagePublishingHistory
142+from lp.soyuz.model.queue import PackageUpload
143 from lp.soyuz.model.reporting import LatestPersonSourcePackageReleaseCache
144 from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
145 from lp.translations.interfaces.potemplate import IPOTemplateSet
146@@ -1461,6 +1463,78 @@
147 transaction.commit()
148
149
150+class BaseKeyMigrator(TunableLoop):
151+
152+ maximum_chunk_size = 5000
153+
154+ def __init__(self, log, abort_time=None):
155+ super(BaseKeyMigrator, self).__init__(
156+ log, abort_time)
157+ state = load_garbo_job_state(self.__class__.__name__) or {}
158+ self.start_at = state.get('next_id', 1)
159+ self.store = IMasterStore(self.klass)
160+
161+ def findObjects(self):
162+ return self.store.find(
163+ self.klass,
164+ self.klass.id >= self.start_at).order_by(
165+ self.klass.id)
166+
167+ def isDone(self):
168+ return (
169+ not getFeatureFlag('gpg.migrator.%s' % self.klass.__name__)
170+ or self.findObjects().is_empty())
171+
172+ def __call__(self, chunk_size):
173+ objs = list(self.findObjects()[:chunk_size])
174+ for obj in objs:
175+ key = getattr(obj, self.fk_attr, None)
176+ if self.fingerprint_attr:
177+ setattr(
178+ obj, self.fingerprint_attr,
179+ key.fingerprint if key else None)
180+ if self.owner_attr:
181+ setattr(
182+ obj, self.owner_attr,
183+ key.owner if key else None)
184+ self.start_at = objs[-1].id + 1
185+ save_garbo_job_state(
186+ self.__class__.__name__, {'next_id': self.start_at})
187+ transaction.commit()
188+
189+
190+class ArchiveKeyMigrator(BaseKeyMigrator):
191+
192+ klass = Archive
193+ fk_attr = 'signing_key'
194+ fingerprint_attr = '_signing_key_fingerprint'
195+ owner_attr = 'signing_key_owner'
196+
197+
198+class PackageUploadKeyMigrator(BaseKeyMigrator):
199+
200+ klass = PackageUpload
201+ fk_attr = 'signing_key'
202+ fingerprint_attr = 'signing_key_fingerprint'
203+ owner_attr = 'signing_key_owner'
204+
205+
206+class SignedCodeOfConductKeyMigrator(BaseKeyMigrator):
207+
208+ klass = SignedCodeOfConduct
209+ fk_attr = 'signingkey'
210+ fingerprint_attr = 'signing_key_fingerprint'
211+ owner_attr = None
212+
213+
214+class SourcePackageReleaseKeyMigrator(BaseKeyMigrator):
215+
216+ klass = SourcePackageRelease
217+ fk_attr = 'dscsigningkey'
218+ fingerprint_attr = 'signing_key_fingerprint'
219+ owner_attr = 'signing_key_owner'
220+
221+
222 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
223 """Abstract base class to run a collection of TunableLoops."""
224 script_name = None # Script name for locking and database user. Override.
225@@ -1709,10 +1783,14 @@
226 """
227 script_name = 'garbo-hourly'
228 tunable_loops = [
229+ ArchiveKeyMigrator,
230 BugHeatUpdater,
231 BugWatchScheduler,
232 DuplicateSessionPruner,
233+ PackageUploadKeyMigrator,
234 RevisionCachePruner,
235+ SignedCodeOfConductKeyMigrator,
236+ SourcePackageReleaseKeyMigrator,
237 UnusedSessionPruner,
238 ]
239 experimental_tunable_loops = []
240
241=== modified file 'lib/lp/scripts/tests/test_garbo.py'
242--- lib/lp/scripts/tests/test_garbo.py 2016-02-06 02:55:51 +0000
243+++ lib/lp/scripts/tests/test_garbo.py 2016-03-03 18:43:25 +0000
244@@ -73,6 +73,7 @@
245 from lp.registry.interfaces.accesspolicy import IAccessPolicySource
246 from lp.registry.interfaces.person import IPersonSet
247 from lp.registry.interfaces.teammembership import TeamMembershipStatus
248+from lp.registry.model.codeofconduct import SignedCodeOfConduct
249 from lp.registry.model.commercialsubscription import CommercialSubscription
250 from lp.registry.model.person import PersonSettings
251 from lp.registry.model.teammembership import TeamMembership
252@@ -128,7 +129,10 @@
253 TestCase,
254 TestCaseWithFactory,
255 )
256-from lp.testing.dbuser import switch_dbuser
257+from lp.testing.dbuser import (
258+ dbuser,
259+ switch_dbuser,
260+ )
261 from lp.testing.layers import (
262 DatabaseLayer,
263 LaunchpadScriptLayer,
264@@ -1390,6 +1394,67 @@
265 self._test_LiveFSFilePruner(
266 'application/octet-stream', 0, expected_count=1)
267
268+ def test_ArchiveKeyMigrator(self):
269+ with dbuser('testadmin'):
270+ key = self.factory.makeGPGKey(self.factory.makePerson())
271+ a1 = self.factory.makeArchive()
272+ a2 = self.factory.makeArchive()
273+ removeSecurityProxy(a2).signing_key = key
274+ self.assertIs(None, a1._signing_key_fingerprint)
275+ self.assertIs(None, a1.signing_key_owner)
276+ self.assertIs(None, a2._signing_key_fingerprint)
277+ self.assertIs(None, a2.signing_key_owner)
278+ with FeatureFixture({'gpg.migrator.Archive': 'on'}):
279+ self.runHourly()
280+ self.assertIs(None, a1._signing_key_fingerprint)
281+ self.assertIs(None, a1.signing_key_owner)
282+ self.assertEqual(key.fingerprint, a2._signing_key_fingerprint)
283+ self.assertEqual(key.owner, a2.signing_key_owner)
284+
285+ def test_PackageUploadKeyMigrator(self):
286+ with dbuser('testadmin'):
287+ key = self.factory.makeGPGKey(self.factory.makePerson())
288+ pu1 = self.factory.makePackageUpload()
289+ pu2 = self.factory.makePackageUpload(signing_key=key)
290+ npu2 = removeSecurityProxy(pu2)
291+ npu2.signing_key_fingerprint = npu2.signing_key_owner = None
292+ self.assertIs(None, pu1.signing_key_fingerprint)
293+ self.assertIs(None, pu1.signing_key_owner)
294+ self.assertIs(None, pu2.signing_key_fingerprint)
295+ self.assertIs(None, pu2.signing_key_owner)
296+ with FeatureFixture({'gpg.migrator.PackageUpload': 'on'}):
297+ self.runHourly()
298+ self.assertIs(None, pu1.signing_key_fingerprint)
299+ self.assertIs(None, pu1.signing_key_owner)
300+ self.assertEqual(key.fingerprint, pu2.signing_key_fingerprint)
301+ self.assertEqual(key.owner, pu2.signing_key_owner)
302+
303+ def test_SignedCodeOfConductKeyMigrator(self):
304+ coc = SignedCodeOfConduct.get(1)
305+ self.assertIs(None, coc.signing_key_fingerprint)
306+ with FeatureFixture({'gpg.migrator.SignedCodeOfConduct': 'on'}):
307+ self.runHourly()
308+ self.assertEqual(
309+ 'ABCDEF0123456789ABCDDCBA0000111112345678',
310+ coc.signing_key_fingerprint)
311+
312+ def test_SourcePackageReleaseKeyMigrator(self):
313+ with dbuser('testadmin'):
314+ key = self.factory.makeGPGKey(self.factory.makePerson())
315+ spr1 = self.factory.makeSourcePackageRelease()
316+ spr2 = self.factory.makeSourcePackageRelease()
317+ removeSecurityProxy(spr2).dscsigningkey = key
318+ self.assertIs(None, spr1.signing_key_fingerprint)
319+ self.assertIs(None, spr1.signing_key_owner)
320+ self.assertIs(None, spr2.signing_key_fingerprint)
321+ self.assertIs(None, spr2.signing_key_owner)
322+ with FeatureFixture({'gpg.migrator.SourcePackageRelease': 'on'}):
323+ self.runHourly()
324+ self.assertIs(None, spr1.signing_key_fingerprint)
325+ self.assertIs(None, spr1.signing_key_owner)
326+ self.assertEqual(key.fingerprint, spr2.signing_key_fingerprint)
327+ self.assertEqual(key.owner, spr2.signing_key_owner)
328+
329 def test_PersonSettingsENFPopulator(self):
330 switch_dbuser('testadmin')
331 store = IMasterStore(PersonSettings)
332
333=== modified file 'lib/lp/soyuz/configure.zcml'
334--- lib/lp/soyuz/configure.zcml 2016-01-08 14:35:52 +0000
335+++ lib/lp/soyuz/configure.zcml 2016-03-03 18:43:25 +0000
336@@ -75,6 +75,7 @@
337 class="lp.soyuz.model.sourcepackagerelease.SourcePackageRelease">
338 <allow
339 interface="lp.soyuz.interfaces.sourcepackagerelease.ISourcePackageRelease"/>
340+ <allow attributes="signing_key_fingerprint signing_key_owner" />
341 <!-- changelog needs to be updated when unembargoing -->
342 <require
343 permission="launchpad.Edit"
344@@ -143,6 +144,8 @@
345 changesfile
346 changes_file_url
347 signing_key
348+ signing_key_fingerprint
349+ signing_key_owner
350 archive
351 sources
352 sourceFileUrls
353@@ -386,7 +389,9 @@
354 set_schema="lp.soyuz.interfaces.archive.IArchiveRestricted"/>
355 <require
356 permission="launchpad.InternalScriptsOnly"
357- set_attributes="distribution signing_key"/>
358+ attributes="signing_key_owner _signing_key_fingerprint"
359+ set_attributes="distribution signing_key signing_key_owner
360+ _signing_key_fingerprint"/>
361 </class>
362 <adapter
363 for="lp.soyuz.interfaces.archive.IArchive"
364
365=== modified file 'lib/lp/soyuz/model/archive.py'
366--- lib/lp/soyuz/model/archive.py 2015-11-26 15:46:38 +0000
367+++ lib/lp/soyuz/model/archive.py 2016-03-03 18:43:25 +0000
368@@ -23,21 +23,22 @@
369 IntCol,
370 StringCol,
371 )
372+from storm.base import Storm
373 from storm.expr import (
374 And,
375+ Count,
376 Desc,
377+ Join,
378 Not,
379 Or,
380 Select,
381 Sum,
382 )
383-from storm.locals import (
384- Count,
385+from storm.properties import (
386 Int,
387- Join,
388- Reference,
389- Storm,
390+ Unicode,
391 )
392+from storm.references import Reference
393 from storm.store import Store
394 from zope.component import (
395 getAdapter,
396@@ -339,6 +340,9 @@
397
398 signing_key = ForeignKey(
399 foreignKey='GPGKey', dbName='signing_key', notNull=False)
400+ signing_key_owner_id = Int(name="signing_key_owner")
401+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
402+ _signing_key_fingerprint = Unicode(name="signing_key_fingerprint")
403
404 relative_build_score = IntCol(
405 dbName='relative_build_score', notNull=True, default=0)
406@@ -2534,6 +2538,9 @@
407 owner=owner, distribution=distribution, name=name,
408 displayname=displayname, description=description,
409 purpose=purpose, publish=publish, signing_key=signing_key,
410+ signing_key_owner=signing_key.owner if signing_key else None,
411+ _signing_key_fingerprint=(
412+ signing_key.fingerprint if signing_key else None),
413 require_virtualized=require_virtualized)
414
415 # Upon creation archives are enabled by default.
416
417=== modified file 'lib/lp/soyuz/model/queue.py'
418--- lib/lp/soyuz/model/queue.py 2016-02-05 16:51:12 +0000
419+++ lib/lp/soyuz/model/queue.py 2016-03-03 18:43:25 +0000
420@@ -186,6 +186,9 @@
421
422 signing_key = ForeignKey(
423 foreignKey='GPGKey', dbName='signing_key', notNull=False)
424+ signing_key_owner_id = Int(name="signing_key_owner")
425+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
426+ signing_key_fingerprint = Unicode()
427
428 package_copy_job_id = Int(name='package_copy_job', allow_none=True)
429 package_copy_job = Reference(package_copy_job_id, 'PackageCopyJob.id')
430
431=== modified file 'lib/lp/soyuz/model/sourcepackagerelease.py'
432--- lib/lp/soyuz/model/sourcepackagerelease.py 2016-02-05 15:16:29 +0000
433+++ lib/lp/soyuz/model/sourcepackagerelease.py 2016-03-03 18:43:25 +0000
434@@ -30,6 +30,7 @@
435 Desc,
436 Int,
437 Reference,
438+ Unicode,
439 )
440 from storm.store import Store
441 from zope.component import getUtility
442@@ -89,6 +90,9 @@
443 dbName='maintainer', foreignKey='Person',
444 storm_validator=validate_public_person, notNull=True)
445 dscsigningkey = ForeignKey(foreignKey='GPGKey', dbName='dscsigningkey')
446+ signing_key_owner_id = Int(name="signing_key_owner")
447+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
448+ signing_key_fingerprint = Unicode()
449 urgency = EnumCol(dbName='urgency', schema=SourcePackageUrgency,
450 default=SourcePackageUrgency.LOW, notNull=True)
451 dateuploaded = UtcDateTimeCol(dbName='dateuploaded', notNull=True,
452
453=== modified file 'lib/lp/soyuz/scripts/gina/handlers.py'
454--- lib/lp/soyuz/scripts/gina/handlers.py 2015-05-20 11:31:11 +0000
455+++ lib/lp/soyuz/scripts/gina/handlers.py 2016-03-03 18:43:25 +0000
456@@ -616,6 +616,8 @@
457 sourcepackagename=name.id,
458 maintainer=maintainer.id,
459 dscsigningkey=key,
460+ signing_key_owner=key.owner if key else None,
461+ signing_key_fingerprint=key.fingerprint if key else None,
462 urgency=ChangesFile.urgency_map[src.urgency],
463 dateuploaded=src.date_uploaded,
464 dsc=src.dsc,