britney

Merge ~mitya57/britney:remove-source-ppa-policy into ~ubuntu-release/britney/+git/britney2-ubuntu:master

  1. Git
  2. lp:~mitya57/britney
  3. remove-source-ppa-policy
  4. Merge into master
Proposed by Dmitry Shachnev
Status: Needs review
Proposed branch: ~mitya57/britney:remove-source-ppa-policy
Merge into: ~ubuntu-release/britney/+git/britney2-ubuntu:master
Diff against target: 876 lines (+19/-478)
8 files modified
britney.py (+0/-2)
dev/null (+0/-404)
tests/__init__.py (+7/-0)
tests/test_autopkgtest.py (+10/-34)
tests/test_email.py (+1/-11)
tests/test_lpexcusebugs.py (+0/-11)
tests/test_sruadtregression.py (+1/-2)
tests/test_yaml.py (+0/-14)
Reviewer Review Type Date Requested Status
Ubuntu Release Team Pending
Review via email: mp+464245@code.launchpad.net

Description of the change

Remove SourcePPAPolicy

It did not always detect the PPA packages correctly, and it did not show which exactly package which is blocking all the other ones.

After discussing on #ubuntu-release, we decided to remove it:
https://irclogs.ubuntu.com/2024/04/13/%23ubuntu-release.html#t16:33

To post a comment you must log in.

Unmerged commits

be5d181... by Dmitry Shachnev

Remove SourcePPAPolicy

It did not always detect the PPA packages correctly, and it did not
show which exactly package which is blocking all the other ones.

After discussing on #ubuntu-release, we decided to remove it:
https://irclogs.ubuntu.com/2024/04/13/%23ubuntu-release.html#t16:33

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/britney.py b/britney.py
2index 8d8f427..2ca3a24 100755
3--- a/britney.py
4+++ b/britney.py
5@@ -220,7 +220,6 @@ from britney2.policies.policy import (AgePolicy,
6 LPBlockBugPolicy,
7 )
8 from britney2.policies.autopkgtest import AutopkgtestPolicy
9-from britney2.policies.sourceppa import SourcePPAPolicy
10 from britney2.policies.sruadtregression import SRUADTRegressionPolicy
11 from britney2.policies.email import EmailPolicy
12 from britney2.policies.lpexcusebugs import LPExcuseBugsPolicy
13@@ -549,7 +548,6 @@ class Britney(object):
14 self._policy_engine.add_policy(BuiltOnBuilddPolicy(self.options, self.suite_info))
15 self._policy_engine.add_policy(LPBlockBugPolicy(self.options, self.suite_info))
16 self._policy_engine.add_policy(LPExcuseBugsPolicy(self.options, self.suite_info))
17- self._policy_engine.add_policy(SourcePPAPolicy(self.options, self.suite_info))
18 self._policy_engine.add_policy(LinuxPolicy(self.options, self.suite_info))
19 add_sruregression_policy = getattr(self.options, 'sruregressionemail_enable', 'no')
20 if add_sruregression_policy in ('yes', 'dry-run'):
21diff --git a/britney2/policies/sourceppa.py b/britney2/policies/sourceppa.py
22deleted file mode 100644
23index bbef00d..0000000
24--- a/britney2/policies/sourceppa.py
25+++ /dev/null
26@@ -1,185 +0,0 @@
27-import os
28-import json
29-import socket
30-import urllib.request
31-import urllib.parse
32-
33-from collections import defaultdict
34-from urllib.error import HTTPError
35-
36-from britney2 import SuiteClass
37-from britney2.policies.rest import Rest
38-from britney2.policies.policy import BasePolicy, PolicyVerdict
39-
40-
41-LAUNCHPAD_URL = "https://api.launchpad.net/1.0/"
42-PRIMARY = LAUNCHPAD_URL + "ubuntu/+archive/primary"
43-INCLUDE = ["~bileto-ppa-service/", "~ci-train-ppa-service/"]
44-EXCLUDE = ["~ci-train-ppa-service/+archive/ubuntu/4810", "~ci-train-ppa-service/+archive/ubuntu/4813", "~ci-train-ppa-service/+archive/ubuntu/4815", "~ci-train-ppa-service/+archive/ubuntu/4816"]
45-
46-
47-class SourcePPAPolicy(BasePolicy, Rest):
48- """Migrate packages copied from same source PPA together
49-
50- This policy will query launchpad to determine what source PPA packages
51- were copied from, and ensure that all packages from the same PPA migrate
52- together.
53- """
54-
55- def __init__(self, options, suite_info):
56- super().__init__(
57- "source-ppa", options, suite_info, {SuiteClass.PRIMARY_SOURCE_SUITE}
58- )
59- self.filename = os.path.join(options.unstable, "SourcePPA")
60- # Dict of dicts; maps pkg name -> pkg version -> source PPA URL
61- self.source_ppas_by_pkg = defaultdict(dict)
62- # Dict of sets; maps source PPA URL -> (set of source names, set of
63- # friends; collected excuses for this ppa)
64- self.excuses_by_source_ppa = defaultdict(set)
65- self.source_ppa_info_by_source_ppa = defaultdict(set)
66- self.britney = None
67- # self.cache contains self.source_ppas_by_pkg from previous run
68- self.cache = {}
69-
70- def lp_get_source_ppa(self, pkg, version):
71- """Ask LP what source PPA pkg was copied from"""
72- cached = self.cache.get(pkg, {}).get(version)
73- if cached is not None:
74- return cached
75-
76- data = self.query_lp_rest_api(
77- "%s/+archive/primary" % self.options.distribution,
78- {
79- "ws.op": "getPublishedSources",
80- "pocket": "Proposed",
81- "source_name": pkg,
82- "version": version,
83- "exact_match": "true",
84- "distro_series": "/%s/%s"
85- % (self.options.distribution, self.options.series),
86- },
87- )
88- try:
89- sourcepub = data["entries"][0]["self_link"]
90- # IndexError means no packages in -proposed matched this name/version,
91- # which is expected to happen when bileto runs britney.
92- except IndexError:
93- self.logger.info(
94- "SourcePPA getPackageUploads IndexError (%s %s)"
95- % (pkg, version)
96- )
97- return "IndexError"
98- data = self.query_lp_rest_api(
99- sourcepub, {"ws.op": "getPublishedBinaries"}
100- )
101- for binary in data["entries"]:
102- link = binary["build_link"] or ""
103- if "/+archive/" in link:
104- ppa, _, buildid = link.partition("/+build/")
105- return ppa
106- return ""
107-
108- def initialise(self, britney):
109- """Load cached source ppa data"""
110- super().initialise(britney)
111- self.britney = britney
112-
113- if os.path.exists(self.filename):
114- with open(self.filename, encoding="utf-8") as data:
115- self.cache = json.load(data)
116- self.logger.info(
117- "Loaded cached source ppa data from %s", self.filename
118- )
119-
120- def apply_src_policy_impl(
121- self,
122- sourceppa_info,
123- item,
124- source_data_tdist,
125- source_data_srcdist,
126- excuse,
127- ):
128- """Reject package if any other package copied from same PPA is invalid"""
129- source_name = item.package
130- accept = excuse.is_valid
131- version = source_data_srcdist.version
132- sourceppa = self.lp_get_source_ppa(source_name, version) or ""
133- verdict = excuse.policy_verdict
134- self.source_ppas_by_pkg[source_name][version] = sourceppa
135- if [team for team in EXCLUDE if team in sourceppa]:
136- return PolicyVerdict.PASS
137- if not [team for team in INCLUDE if team in sourceppa]:
138- return PolicyVerdict.PASS
139-
140- # check for a force hint; we have to check here in addition to
141- # checking in britney.py, otherwise /this/ package will later be
142- # considered valid candidate but all the /others/ from the ppa will
143- # be invalidated via this policy and not fixed by the force hint.
144- forces = self.hints.search(
145- "force", package=source_name, version=source_data_srcdist.version
146- )
147- if forces:
148- excuse.dontinvalidate = True
149- changed_state = excuse.force()
150- if changed_state:
151- excuse.addhtml(
152- "Should ignore, but forced by %s" % (forces[0].user)
153- )
154- accept = True
155-
156- shortppa = sourceppa.replace(LAUNCHPAD_URL, "")
157- sourceppa_info[source_name] = shortppa
158-
159- if not excuse.is_valid:
160- self.logger.info(
161- "sourceppa: processing %s, which is invalid, will invalidate set",
162- source_name,
163- )
164- else:
165- # Check for other packages that might invalidate this one
166- for friend_exc in self.excuses_by_source_ppa[sourceppa]:
167- sourceppa_info[friend_exc.item.package] = shortppa
168- if not friend_exc.is_valid:
169- self.logger.info(
170- "sourceppa: processing %s, found invalid grouped package %s, will invalidate set"
171- % (source_name, friend_exc.name)
172- )
173- accept = False
174- break
175-
176- self.excuses_by_source_ppa[sourceppa].add(excuse)
177-
178- if not accept:
179- # Invalidate all packages in this source ppa
180- for friend_exc in self.excuses_by_source_ppa[sourceppa]:
181- self.logger.info("friend: %s", friend_exc.name)
182- sourceppa_info[friend_exc.item.package] = shortppa
183- if friend_exc.is_valid:
184- if friend_exc == excuse:
185- verdict = PolicyVerdict.REJECTED_WAITING_FOR_ANOTHER_ITEM
186- else:
187- friend_exc.invalidate_externally(
188- PolicyVerdict.REJECTED_WAITING_FOR_ANOTHER_ITEM
189- )
190- friend_exc.addreason("source-ppa")
191- self.logger.info(
192- "sourceppa: ... invalidating %s due to the above (ppa: %s), %s"
193- % (friend_exc.name, shortppa, sourceppa_info)
194- )
195- friend_exc.addinfo("Grouped with PPA %s" % shortppa)
196-
197- for friend_exc in self.excuses_by_source_ppa[sourceppa]:
198- try:
199- friend_exc.policy_info["source-ppa"].update(sourceppa_info)
200- except KeyError:
201- friend_exc.policy_info["source-ppa"] = sourceppa_info.copy()
202-
203- return verdict
204-
205- def save_state(self, britney):
206- """Write source ppa data to disk"""
207- tmp = self.filename + ".tmp"
208- with open(tmp, "w", encoding="utf-8") as data:
209- json.dump(self.source_ppas_by_pkg, data)
210- os.rename(tmp, self.filename)
211- self.logger.info("Wrote source ppa data to %s" % self.filename)
212diff --git a/tests/__init__.py b/tests/__init__.py
213index eaa9436..2d7af51 100644
214--- a/tests/__init__.py
215+++ b/tests/__init__.py
216@@ -445,3 +445,10 @@ EMAIL_ENABLE = yes
217 with open(hints_path, 'a') as fd:
218 fd.write(content)
219 fd.write('\n')
220+
221+
222+class FakeOptions:
223+ distribution = "testbuntu"
224+ series = "zazzy"
225+ unstable = "/tmp"
226+ verbose = False
227diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py
228index 5c7bb72..36eef3a 100644
229--- a/tests/test_autopkgtest.py
230+++ b/tests/test_autopkgtest.py
231@@ -56,31 +56,18 @@ class TestAutopkgtestBase(TestBase):
232 else:
233 sys.stdout.write(line)
234
235- # Set up sourceppa cache for testing
236- self.sourceppa_cache = {
237- 'gcc-5': {'2': ''},
238- 'gcc-snapshot': {'2': ''},
239- 'green': {'2': '', '1.1': '', '3': ''},
240- 'lightgreen': {'2': '', '1.1~beta': '', '3': ''},
241- 'linux-meta-64only': {'1': ''},
242- 'linux-meta-lts-grumpy': {'1': ''},
243- 'linux-meta': {'0.2': '', '1': '', '2': ''},
244- 'linux': {'2': ''},
245- 'newgreen': {'2': ''},
246+ self.email_cache = {
247+ 'gcc-5': {'2': True},
248+ 'gcc-snapshot': {'2': True},
249+ 'green': {'1.1': True, '2': True, '3': True},
250+ 'lightgreen': {'1.1~beta': True, '2': True, '3': True},
251+ 'linux': {'2': True},
252+ 'linux-meta': {'0.2': True, '1': True, '2': True},
253+ 'linux-meta-64only': {'1': True},
254+ 'linux-meta-lts-grumpy': {'1': True},
255+ 'newgreen': {'2': True},
256 }
257
258- self.email_cache = {}
259- for pkg, vals in self.sourceppa_cache.items():
260- for version, empty in vals.items():
261- self.email_cache.setdefault(pkg, {})
262- self.email_cache[pkg][version] = True
263-
264- self.email_cache = {}
265- for pkg, vals in self.sourceppa_cache.items():
266- for version, empty in vals.items():
267- self.email_cache.setdefault(pkg, {})
268- self.email_cache[pkg][version] = True
269-
270 # create mock Swift server (but don't start it yet, as tests first need
271 # to poke in results)
272 self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085)
273@@ -199,17 +186,9 @@ class TestAutopkgtestBase(TestBase):
274 '''
275 for (pkg, fields, testsuite) in unstable_add:
276 self.data.add(pkg, True, fields, True, testsuite)
277- self.sourceppa_cache.setdefault(pkg, {})
278- if fields['Version'] not in self.sourceppa_cache[pkg]:
279- self.sourceppa_cache[pkg][fields['Version']] = ''
280 self.email_cache.setdefault(pkg, {})
281 self.email_cache[pkg][fields['Version']] = True
282
283- # Set up sourceppa cache for testing
284- sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA')
285- with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa:
286- sourceppa.write(json.dumps(self.sourceppa_cache))
287-
288 email_path = os.path.join(self.data.dirs[True], 'EmailCache')
289 with open(email_path, 'w', encoding='utf-8') as email:
290 email.write(json.dumps(self.email_cache))
291@@ -333,7 +312,6 @@ class AT(TestAutopkgtestBase):
292 '''
293
294 self.data.add_default_packages()
295- self.sourceppa_cache['purple'] = {'2': ''}
296
297 # The package has passed before on i386
298 self.set_results({'autopkgtest-testing': {
299@@ -854,8 +832,6 @@ class AT(TestAutopkgtestBase):
300
301 self.data.add_default_packages(green=False)
302
303- self.sourceppa_cache['lime'] = {'1': ''}
304-
305 self.data.add_src('lime', True, {'Version': '1', 'Testsuite': 'autopkgtest'})
306 exc = self.run_it(
307 # unbuilt unstable version
308diff --git a/tests/test_email.py b/tests/test_email.py
309index 03d4305..1f21e0e 100755
310--- a/tests/test_email.py
311+++ b/tests/test_email.py
312@@ -23,8 +23,7 @@ sys.path.insert(0, PROJECT_DIR)
313 from britney2.policies.policy import PolicyVerdict
314 from britney2.policies.email import EmailPolicy, person_chooser, address_chooser
315
316-from tests.test_sourceppa import FakeOptions
317-from tests import TestBase
318+from tests import FakeOptions, TestBase
319 from tests.mock_smtpd import FakeSMTPServer
320
321 # Example of a direct upload by core dev: openstack-doc-tools 1.5.0-0ubuntu1
322@@ -396,7 +395,6 @@ class ET(TestBase):
323 self.age_file = os.path.join(self.data.dirs[False], "Dates")
324 self.urgency_file = os.path.join(self.data.dirs[False], "Urgency")
325 self.email_cache_file = os.path.join(self.data.dirs[True], "EmailCache")
326- self.sourceppa_cache = {}
327 self.email_cache = {}
328
329 self.data.add("libc6", False)
330@@ -414,9 +412,6 @@ class ET(TestBase):
331 ET.smtpd.emails.clear()
332 for (pkg, fields, daysold, emails) in unstable_add:
333 self.data.add(pkg, True, fields, True, None)
334- self.sourceppa_cache.setdefault(pkg, {})
335- if fields["Version"] not in self.sourceppa_cache[pkg]:
336- self.sourceppa_cache[pkg][fields["Version"]] = ""
337 with open(self.age_file, "w") as f:
338 import time
339
340@@ -428,11 +423,6 @@ class ET(TestBase):
341 d[pkg][fields["Version"]] = (emails, 0)
342 f.write(json.dumps(d))
343
344- # Set up sourceppa cache for testing
345- sourceppa_path = os.path.join(self.data.dirs[True], "SourcePPA")
346- with open(sourceppa_path, "w", encoding="utf-8") as sourceppa:
347- sourceppa.write(json.dumps(self.sourceppa_cache))
348-
349 (excuses_yaml, excuses_html, out) = self.run_britney()
350
351 # convert excuses to source indexed dict
352diff --git a/tests/test_lpexcusebugs.py b/tests/test_lpexcusebugs.py
353index e6bce78..1b14014 100755
354--- a/tests/test_lpexcusebugs.py
355+++ b/tests/test_lpexcusebugs.py
356@@ -10,7 +10,6 @@ from collections import defaultdict
357
358 import calendar
359 import fileinput
360-import json
361 import os
362 import pprint
363 import sys
364@@ -25,7 +24,6 @@ sys.path.insert(0, PROJECT_DIR)
365 from britney2.policies.policy import PolicyVerdict
366 from britney2.policies.email import EmailPolicy, person_chooser, address_chooser
367
368-from tests.test_sourceppa import FakeOptions
369 from tests import TestBase
370
371
372@@ -63,7 +61,6 @@ class ET(TestBase):
373 ):
374 sys.stdout.write(line)
375 self.excuse_bugs_file = os.path.join(self.data.dirs[True], "ExcuseBugs")
376- self.sourceppa_cache = {}
377
378 self.data.add("libc6", False)
379
380@@ -80,19 +77,11 @@ class ET(TestBase):
381 """
382 for (pkg, fields, bugs) in unstable_add:
383 self.data.add(pkg, True, fields, True, None)
384- self.sourceppa_cache.setdefault(pkg, {})
385- if fields["Version"] not in self.sourceppa_cache[pkg]:
386- self.sourceppa_cache[pkg][fields["Version"]] = ""
387 print("Writing to %s" % self.excuse_bugs_file)
388 with open(self.excuse_bugs_file, "w") as f:
389 for (bug, ts) in bugs:
390 f.write("%s %s %s" % (pkg, bug, ts))
391
392- # Set up sourceppa cache for testing
393- sourceppa_path = os.path.join(self.data.dirs[True], "SourcePPA")
394- with open(sourceppa_path, "w", encoding="utf-8") as sourceppa:
395- sourceppa.write(json.dumps(self.sourceppa_cache))
396-
397 (excuses_yaml, excuses_html, out) = self.run_britney()
398
399 bugs_blocked_by = []
400diff --git a/tests/test_sourceppa.py b/tests/test_sourceppa.py
401deleted file mode 100755
402index 61c6c5a..0000000
403--- a/tests/test_sourceppa.py
404+++ /dev/null
405@@ -1,404 +0,0 @@
406-#!/usr/bin/python3
407-# (C) 2016 Canonical Ltd.
408-#
409-# This program is free software; you can redistribute it and/or modify
410-# it under the terms of the GNU General Public License as published by
411-# the Free Software Foundation; either version 2 of the License, or
412-# (at your option) any later version.
413-
414-import json
415-import os
416-import sys
417-import unittest
418-from unittest.mock import DEFAULT, patch
419-
420-PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
421-sys.path.insert(0, PROJECT_DIR)
422-
423-from britney2 import Suite, SuiteClass
424-from britney2.excuse import Excuse
425-from britney2.hints import HintCollection
426-from britney2.migrationitem import MigrationItem
427-from britney2.policies.policy import PolicyEngine, PolicyVerdict
428-from britney2.policies.sourceppa import LAUNCHPAD_URL, SourcePPAPolicy
429-
430-# We want to reuse run_it
431-from tests.test_autopkgtest import TestAutopkgtestBase, tr
432-
433-
434-CACHE_FILE = os.path.join(PROJECT_DIR, "tests", "data", "sourceppa.json")
435-
436-
437-class FakeOptions:
438- distribution = "testbuntu"
439- series = "zazzy"
440- unstable = "/tmp"
441- verbose = False
442-
443-
444-class FakeExcuse(Excuse):
445- def __init__(self, name, suite):
446- self.item = MigrationItem(package=name, version="2.0", suite=suite)
447- Excuse.__init__(self, self.item)
448- self.policy_verdict = PolicyVerdict.PASS
449-
450-
451-SOURCE_SUITE = Suite(SuiteClass.PRIMARY_SOURCE_SUITE, "fakename", "fakepath")
452-
453-PAL = FakeExcuse("pal", SOURCE_SUITE)
454-BUDDY = FakeExcuse("buddy", SOURCE_SUITE)
455-FRIEND = FakeExcuse("friend", SOURCE_SUITE)
456-NOPPA = FakeExcuse("noppa", SOURCE_SUITE)
457-
458-
459-class FakeBritney:
460- def __init__(self):
461- self._policy = SourcePPAPolicy(FakeOptions, {})
462- self._policy.filename = CACHE_FILE
463- self._policy_engine = PolicyEngine()
464- self._policy_engine.add_policy(self._policy)
465- self._policy_engine.initialise(self, HintCollection())
466-
467-
468-class FakeData:
469- version = "2.0"
470-
471-
472-class T(unittest.TestCase):
473- maxDiff = None
474-
475- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
476- def test_lp_rest_api_no_entries(self, urlopen):
477- """Don't explode if LP reports no entries match pkg/version"""
478- context = urlopen.return_value.__enter__.return_value
479- context.getcode.return_value = 200
480- context.read.return_value = b'{"entries": []}'
481- pol = SourcePPAPolicy(FakeOptions, {})
482- self.assertEqual(pol.lp_get_source_ppa("hello", "1.0"), "IndexError")
483-
484- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
485- def test_lp_rest_api_no_source_ppa(self, urlopen):
486- """Identify when package has no source PPA"""
487- context = urlopen.return_value.__enter__.return_value
488- context.getcode.return_value = 200
489- context.read.return_value = b'{"entries": [{"self_link": "https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/12345", "build_link": "https://api.launchpad.net/1.0/ubuntu/+source/gcc-5/5.4.1-7ubuntu1/+build/12066956", "other_stuff": "ignored"}]}'
490- pol = SourcePPAPolicy(FakeOptions, {})
491- self.assertEqual(pol.lp_get_source_ppa("hello", "1.0"), "")
492-
493- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
494- def test_lp_rest_api_with_source_ppa(self, urlopen):
495- """Identify source PPA"""
496- context = urlopen.return_value.__enter__.return_value
497- context.getcode.return_value = 200
498- context.read.return_value = b'{"entries": [{"self_link": "https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/12345", "build_link": "https://api.launchpad.net/1.0/~ci-train-ppa-service/+archive/ubuntu/2516/+build/12063031", "other_stuff": "ignored"}]}'
499- pol = SourcePPAPolicy(FakeOptions, {})
500- self.assertEqual(
501- pol.lp_get_source_ppa("hello", "1.0"),
502- "https://api.launchpad.net/1.0/~ci-train-ppa-service/+archive/ubuntu/2516",
503- )
504-
505- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
506- def test_lp_rest_api_errors(self, urlopen):
507- """Report errors instead of swallowing them"""
508- context = urlopen.return_value.__enter__.return_value
509- context.getcode.return_value = 500
510- context.read.return_value = b""
511- pol = SourcePPAPolicy(FakeOptions, {})
512- with self.assertRaisesRegex(ConnectionError, "HTTP 500"):
513- pol.lp_get_source_ppa("hello", "1.0")
514- # Yes, I have really seen "success with no json returned" in the wild
515- context.getcode.return_value = 200
516- context.read.return_value = b""
517- with self.assertRaisesRegex(ValueError, "Expecting value"):
518- pol.lp_get_source_ppa("hello", "1.0")
519-
520- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
521- def test_lp_rest_api_timeout(self, urlopen):
522- """If we get a timeout connecting to LP, we try 5 times"""
523- import socket
524-
525- # test that we're retried 5 times on timeout
526- urlopen.side_effect = socket.timeout
527- pol = SourcePPAPolicy(FakeOptions, {})
528- with self.assertRaises(socket.timeout):
529- pol.lp_get_source_ppa("hello", "1.0")
530- self.assertEqual(urlopen.call_count, 5)
531-
532- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
533- def test_lp_rest_api_unavailable(self, urlopen):
534- """If we get a 503 connecting to LP, we try 5 times"""
535- from urllib.error import HTTPError
536-
537- # test that we're retried 5 times on 503
538- urlopen.side_effect = HTTPError(
539- None, 503, "Service Temporarily Unavailable", None, None
540- )
541- pol = SourcePPAPolicy(FakeOptions, {})
542- with self.assertRaises(HTTPError):
543- pol.lp_get_source_ppa("hello", "1.0")
544- self.assertEqual(urlopen.call_count, 5)
545-
546- @patch("britney2.policies.sourceppa.urllib.request.urlopen")
547- def test_lp_rest_api_flaky(self, urlopen):
548- """If we get a 503, then a 200, we get the right result"""
549- from urllib.error import HTTPError
550-
551- def fail_for_a_bit():
552- for i in range(3):
553- yield HTTPError(
554- None, 503, "Service Temporarily Unavailable", None, None
555- )
556- while True:
557- yield DEFAULT
558-
559- context = urlopen.return_value.__enter__.return_value
560- context.getcode.return_value = 200
561- context.read.return_value = b'{"entries": [{"self_link": "https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/12345", "build_link": "https://api.launchpad.net/1.0/~ci-train-ppa-service/+archive/ubuntu/2516/+build/12063031", "other_stuff": "ignored"}]}'
562- urlopen.side_effect = fail_for_a_bit()
563- pol = SourcePPAPolicy(FakeOptions, {})
564- pol.lp_get_source_ppa("hello", "1.0")
565- self.assertEqual(urlopen.call_count, 5)
566- self.assertEqual(
567- pol.lp_get_source_ppa("hello", "1.0"),
568- "https://api.launchpad.net/1.0/~ci-train-ppa-service/+archive/ubuntu/2516",
569- )
570-
571- def test_approve_ppa(self):
572- """Approve packages by their PPA."""
573- shortppa = "~ci-train-ppa-service/+archive/NNNN"
574- brit = FakeBritney()
575- for excuse in (PAL, BUDDY, FRIEND, NOPPA):
576- brit._policy_engine.apply_src_policies(
577- excuse.item, FakeData, FakeData, excuse
578- )
579- self.assertEqual(excuse.policy_verdict, PolicyVerdict.PASS)
580- output = FRIEND.policy_info["source-ppa"]
581- self.assertDictContainsSubset(
582- dict(pal=shortppa, buddy=shortppa, friend=shortppa), output
583- )
584-
585- def test_ignore_ppa(self):
586- """Ignore packages in non-bileto PPAs."""
587- shortppa = "~kernel-or-whatever/+archive/ppa"
588- brit = FakeBritney()
589- for name, versions in brit._policy.cache.items():
590- for version in versions:
591- brit._policy.cache[name][version] = shortppa
592- for excuse in (PAL, BUDDY, FRIEND, NOPPA):
593- brit._policy_engine.apply_src_policies(
594- excuse.item, FakeData, FakeData, excuse
595- )
596- self.assertEqual(excuse.policy_verdict, PolicyVerdict.PASS)
597- output = FRIEND.policy_info["source-ppa"]
598- self.assertEqual(output, {"verdict": "PASS"})
599-
600- def test_reject_ppa(self):
601- """Reject packages by their PPA."""
602- shortppa = "~ci-train-ppa-service/+archive/NNNN"
603- brit = FakeBritney()
604- excuse = BUDDY
605- excuse.policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY
606- # Just buddy is invalid but whole ppa fails
607-
608- # This one passes because the rejection isn't known yet
609- excuse = PAL
610- brit._policy_engine.apply_src_policies(
611- excuse.item, FakeData, FakeData, excuse
612- )
613- self.assertEqual(excuse.policy_verdict, PolicyVerdict.PASS)
614- # This one fails because it is itself invalid.
615- excuse = BUDDY
616- brit._policy_engine.apply_src_policies(
617- excuse.item, FakeData, FakeData, excuse
618- )
619- self.assertEqual(
620- excuse.policy_verdict, PolicyVerdict.REJECTED_PERMANENTLY
621- )
622- # This one fails because buddy failed before it.
623- excuse = FRIEND
624- brit._policy_engine.apply_src_policies(
625- excuse.item, FakeData, FakeData, excuse
626- )
627- self.assertEqual(
628- excuse.policy_verdict,
629- PolicyVerdict.REJECTED_WAITING_FOR_ANOTHER_ITEM,
630- )
631- # 'noppa' not from PPA so not rejected
632- excuse = NOPPA
633- brit._policy_engine.apply_src_policies(
634- excuse.item, FakeData, FakeData, excuse
635- )
636- self.assertEqual(excuse.policy_verdict, PolicyVerdict.PASS)
637- # All are rejected however
638- for excuse in (PAL, BUDDY, FRIEND):
639- self.assertFalse(excuse.is_valid)
640- self.assertDictEqual(
641- brit._policy.excuses_by_source_ppa,
642- {LAUNCHPAD_URL + shortppa: {PAL, BUDDY, FRIEND}},
643- )
644- output = FRIEND.policy_info["source-ppa"]
645- self.assertDictEqual(
646- dict(
647- pal=shortppa,
648- buddy=shortppa,
649- friend=shortppa,
650- verdict="REJECTED_WAITING_FOR_ANOTHER_ITEM",
651- ),
652- output,
653- )
654-
655- output = BUDDY.policy_info["source-ppa"]
656- self.assertDictEqual(
657- dict(
658- pal=shortppa,
659- buddy=shortppa,
660- friend=shortppa,
661- verdict="REJECTED_PERMANENTLY"
662- ),
663- output,
664- )
665-
666-
667-class AT(TestAutopkgtestBase):
668- """ Integration tests for source ppa grouping """
669-
670- def test_sourceppa_policy(self):
671- """Packages from same source PPA get rejected for failed peer policy"""
672-
673- self.data.add_default_packages(green=False)
674-
675- ppa = "devel/~ci-train-ppa-service/+archive/NNNN"
676- self.sourceppa_cache["green"] = {"2": ppa}
677- self.sourceppa_cache["red"] = {"2": ppa}
678- with open(
679- os.path.join(self.data.path, "data/unstable/Blocks"), "w"
680- ) as f:
681- f.write("green 12345 1471505000\ndarkgreen 98765 1471500000\n")
682-
683- exc = self.run_it(
684- [
685- ("green", {"Version": "2"}, "autopkgtest"),
686- ("red", {"Version": "2"}, "autopkgtest"),
687- ("gcc-5", {}, "autopkgtest"),
688- ],
689- {
690- "green": (
691- False,
692- {
693- "green": {
694- "i386": "RUNNING-ALWAYSFAIL",
695- "amd64": "RUNNING-ALWAYSFAIL",
696- }
697- },
698- ),
699- "red": (
700- False,
701- {
702- "red": {
703- "i386": "RUNNING-ALWAYSFAIL",
704- "amd64": "RUNNING-ALWAYSFAIL",
705- }
706- },
707- ),
708- "gcc-5": (True, {}),
709- },
710- {"green": [("reason", "block")], "red": [("reason", "source-ppa")]},
711- )[1]
712- self.assertEqual(
713- exc["red"]["policy_info"]["source-ppa"],
714- {
715- "red": ppa,
716- "green": ppa,
717- "verdict": "REJECTED_WAITING_FOR_ANOTHER_ITEM",
718- },
719- )
720-
721- with open(os.path.join(self.data.path, "data/unstable/SourcePPA")) as f:
722- res = json.load(f)
723- self.assertEqual(
724- res,
725- {"red": {"2": ppa}, "green": {"2": ppa}, "gcc-5": {"1": ""}},
726- )
727-
728- def test_sourceppa_missingbuild(self):
729- """Packages from same source PPA get rejected for failed peer FTBFS"""
730-
731- self.data.add_default_packages(green=False)
732-
733- ppa = "devel/~ci-train-ppa-service/+archive/ZZZZ"
734- self.sourceppa_cache["green"] = {"2": ppa}
735- self.sourceppa_cache["red"] = {"2": ppa}
736-
737- self.data.add_src(
738- "green", True, {"Version": "2", "Testsuite": "autopkgtest"}
739- )
740- self.data.add(
741- "libgreen1",
742- True,
743- {"Version": "2", "Source": "green", "Architecture": "i386"},
744- add_src=False,
745- )
746- self.data.add(
747- "green",
748- True,
749- {"Depends": "libc6 (>= 0.9), libgreen1", "Conflicts": "blue"},
750- testsuite="autopkgtest",
751- add_src=False,
752- )
753-
754- self.swift.set_results(
755- {
756- "autopkgtest-testing": {
757- "testing/i386/d/darkgreen/20150101_100000@": (
758- 0,
759- "darkgreen 1",
760- tr("green/2"),
761- ),
762- "testing/i386/l/lightgreen/20150101_100100@": (
763- 0,
764- "lightgreen 1",
765- tr("green/2"),
766- ),
767- "testing/i386/g/green/20150101_100200@": (
768- 0,
769- "green 2",
770- tr("green/2"),
771- ),
772- }
773- }
774- )
775-
776- exc = self.run_it(
777- [("red", {"Version": "2"}, "autopkgtest")],
778- {"green": (False, {}), "red": (False, {})},
779- {
780- "green": [
781- (
782- "missing-builds",
783- {
784- "on-architectures": [
785- "amd64",
786- "arm64",
787- "armhf",
788- "powerpc",
789- "ppc64el",
790- ],
791- "on-unimportant-architectures": [],
792- },
793- )
794- ],
795- "red": [("reason", "source-ppa")],
796- },
797- )[1]
798- self.assertEqual(
799- exc["red"]["policy_info"]["source-ppa"],
800- {
801- "red": ppa,
802- "green": ppa,
803- "verdict": "REJECTED_WAITING_FOR_ANOTHER_ITEM",
804- },
805- )
806-
807-
808-if __name__ == "__main__":
809- unittest.main()
810diff --git a/tests/test_sruadtregression.py b/tests/test_sruadtregression.py
811index 121bacf..2e1dc80 100755
812--- a/tests/test_sruadtregression.py
813+++ b/tests/test_sruadtregression.py
814@@ -20,7 +20,6 @@ sys.path.insert(0, PROJECT_DIR)
815
816 from britney2.policies.policy import PolicyVerdict
817 from britney2.policies.sruadtregression import SRUADTRegressionPolicy
818-from tests.test_sourceppa import FakeBritney
819
820
821 FAKE_CHANGES = b"""Format: 1.8
822@@ -501,7 +500,7 @@ class T(unittest.TestCase):
823 with open(state_path, "w") as f:
824 json.dump(state, f)
825 pol = SRUADTRegressionPolicy(options, {})
826- pol.initialise(FakeBritney())
827+ pol.initialise(None)
828 # Check if the stale packages got purged and others not
829 expected_state = {
830 "testbuntu": {
831diff --git a/tests/test_yaml.py b/tests/test_yaml.py
832index d27d6e0..796eba6 100755
833--- a/tests/test_yaml.py
834+++ b/tests/test_yaml.py
835@@ -7,7 +7,6 @@
836 # (at your option) any later version.
837
838 import fileinput
839-import json
840 import os
841 import pprint
842 import sys
843@@ -42,13 +41,8 @@ class YamlTest(TestBase):
844 sys.stdout.write(line)
845
846 self.data.add('libc6', False)
847- self.sourceppa_cache = {}
848
849 self.email_cache = {}
850- for pkg, vals in self.sourceppa_cache.items():
851- for version, empty in vals.items():
852- self.email_cache.setdefault(pkg, {})
853- self.email_cache[pkg][version] = True
854
855 # create mock Swift server (but don't start it yet, as tests first need
856 # to poke in results)
857@@ -72,19 +66,11 @@ class YamlTest(TestBase):
858 'age-policy-dates')
859 for (pkg, fields, daysold) in unstable_add:
860 self.data.add(pkg, True, fields, True, None)
861- self.sourceppa_cache.setdefault(pkg, {})
862- if fields['Version'] not in self.sourceppa_cache[pkg]:
863- self.sourceppa_cache[pkg][fields['Version']] = ''
864 with open(age_file, 'w') as f:
865 import time
866 do = time.time() - (60 * 60 * 24 * daysold)
867 f.write('%s %s %d' % (pkg, fields['Version'], do))
868
869- # Set up sourceppa cache for testing
870- sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA')
871- with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa:
872- sourceppa.write(json.dumps(self.sourceppa_cache))
873-
874 (excuses_yaml, excuses_html, out) = self.run_britney()
875
876 # convert excuses to source indexed dict

Subscribers

People subscribed via source and target branches