Launchpad itself

Merge lp:~wgrant/launchpad/refactor-nuf-creation into lp:launchpad

  1. refactor-nuf-creation
  2. Merge into devel
Proposed by William Grant
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11275
Proposed branch: lp:~wgrant/launchpad/refactor-nuf-creation
Merge into: lp:launchpad
Diff against target: 315 lines (+124/-53)
4 files modified
lib/lp/archiveuploader/changesfile.py (+58/-47)
lib/lp/archiveuploader/tests/test_changesfile.py (+54/-0)
lib/lp/archiveuploader/tests/test_utils.py (+5/-0)
lib/lp/archiveuploader/utils.py (+7/-6)
To merge this branch: bzr merge lp:~wgrant/launchpad/refactor-nuf-creation
Reviewer Review Type Date Requested Status
Jelmer Vernooij (community) code Approve
Review via email: mp+30851@code.launchpad.net

Commit message

Factor out and test part of ChangesFile.processFiles, so we can use it later in NascentUpload tests.

Description of the change

This branch factors out and tests part of ChangesFile.processFiles. This will be used by a later branch to enable NascentUpload tests without having to create real uploads.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) wrote :

Nice work.

Rather than returning None it seems more sensible to me to raise some sort of exception. With that change, r=me.

review: Needs Fixing (code)
Revision history for this message
Jelmer Vernooij (jelmer) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archiveuploader/changesfile.py'
2--- lib/lp/archiveuploader/changesfile.py 2009-10-12 09:55:31 +0000
3+++ lib/lp/archiveuploader/changesfile.py 2010-08-01 07:06:04 +0000
4@@ -10,21 +10,30 @@
5 __metaclass__ = type
6
7 __all__ = [
8- 'ChangesFile'
9+ 'CannotDetermineFileTypeError',
10+ 'ChangesFile',
11+ 'determine_file_class_and_name',
12 ]
13
14 import os
15
16 from lp.archiveuploader.dscfile import DSCFile, SignableTagFile
17 from lp.archiveuploader.nascentuploadfile import (
18- BaseBinaryUploadFile, CustomUploadFile, DdebBinaryUploadFile,
19+ BaseBinaryUploadFile, CustomUploadFile, DdebBinaryUploadFile,
20 DebBinaryUploadFile, SourceUploadFile, UdebBinaryUploadFile,
21 UploadError, UploadWarning, splitComponentAndSection)
22 from lp.archiveuploader.utils import (
23+ determine_binary_file_type, determine_source_file_type,
24 re_isadeb, re_issource, re_changes_file_name)
25 from lp.archiveuploader.tagfiles import (
26 parse_tagfile, TagFileParseError)
27-from canonical.launchpad.interfaces import SourcePackageUrgency
28+from lp.registry.interfaces.sourcepackage import (SourcePackageFileType,
29+ SourcePackageUrgency)
30+from lp.soyuz.interfaces.binarypackagerelease import BinaryPackageFileType
31+
32+
33+class CannotDetermineFileTypeError(Exception):
34+ """The type of the given file could not be determined."""
35
36
37 class ChangesFile(SignableTagFile):
38@@ -42,7 +51,7 @@
39 "medium": SourcePackageUrgency.MEDIUM,
40 "high": SourcePackageUrgency.HIGH,
41 "critical": SourcePackageUrgency.EMERGENCY,
42- "emergency": SourcePackageUrgency.EMERGENCY
43+ "emergency": SourcePackageUrgency.EMERGENCY,
44 }
45
46 dsc = None
47@@ -103,7 +112,7 @@
48
49 def checkFileName(self):
50 """Make sure the changes file name is well-formed.
51-
52+
53 Please note: for well-formed changes file names the `filename_archtag`
54 property will be set appropriately.
55 """
56@@ -152,8 +161,9 @@
57 def processFiles(self):
58 """Build objects for each file mentioned in this changesfile.
59
60- This method is an error generator, i.e, it returns an iterator over all
61- exceptions that are generated while processing all mentioned files.
62+ This method is an error generator, i.e, it returns an iterator over
63+ all exceptions that are generated while processing all mentioned
64+ files.
65 """
66 files = []
67 for fileline in self._dict['files'].strip().split("\n"):
68@@ -161,8 +171,6 @@
69 # CHECKSUM SIZE [COMPONENT/]SECTION PRIORITY FILENAME
70 digest, size, component_and_section, priority_name, filename = (
71 fileline.strip().split())
72- source_match = re_issource.match(filename)
73- binary_match = re_isadeb.match(filename)
74 filepath = os.path.join(self.dirname, filename)
75 try:
76 if self.isCustom(component_and_section):
77@@ -172,42 +180,22 @@
78 file_instance = CustomUploadFile(
79 filepath, digest, size, component_and_section,
80 priority_name, self.policy, self.logger)
81- elif source_match:
82- package = source_match.group(1)
83- if filename.endswith("dsc"):
84- file_instance = DSCFile(
85- filepath, digest, size, component_and_section,
86- priority_name, package, self.version, self,
87- self.policy, self.logger)
88- # Store the DSC because it is very convenient
89+ else:
90+ try:
91+ package, cls = determine_file_class_and_name(filename)
92+ except CannotDetermineFileTypeError:
93+ yield UploadError(
94+ "Unable to identify file %s (%s) in changes."
95+ % (filename, component_and_section))
96+ continue
97+
98+ file_instance = cls(
99+ filepath, digest, size, component_and_section,
100+ priority_name, package, self.version, self,
101+ self.policy, self.logger)
102+
103+ if cls == DSCFile:
104 self.dsc = file_instance
105- else:
106- file_instance = SourceUploadFile(
107- filepath, digest, size, component_and_section,
108- priority_name, package, self.version, self,
109- self.policy, self.logger)
110- elif binary_match:
111- package = binary_match.group(1)
112- if filename.endswith("udeb"):
113- file_instance = UdebBinaryUploadFile(
114- filepath, digest, size, component_and_section,
115- priority_name, package, self.version, self,
116- self.policy, self.logger)
117- elif filename.endswith("ddeb"):
118- file_instance = DdebBinaryUploadFile(
119- filepath, digest, size, component_and_section,
120- priority_name, package, self.version, self,
121- self.policy, self.logger)
122- else:
123- file_instance = DebBinaryUploadFile(
124- filepath, digest, size, component_and_section,
125- priority_name, package, self.version, self,
126- self.policy, self.logger)
127- else:
128- yield UploadError(
129- "Unable to identify file %s (%s) in changes."
130- % (filename, component_and_section))
131- continue
132 except UploadError, error:
133 yield error
134 else:
135@@ -218,8 +206,8 @@
136 def verify(self):
137 """Run all the verification checks on the changes data.
138
139- This method is an error generator, i.e, it returns an iterator over all
140- exceptions that are generated while verifying the changesfile
141+ This method is an error generator, i.e, it returns an iterator over
142+ all exceptions that are generated while verifying the changesfile
143 consistency.
144 """
145 self.logger.debug("Verifying the changes file.")
146@@ -228,7 +216,7 @@
147 yield UploadError("No files found in the changes")
148
149 raw_urgency = self._dict['urgency'].lower()
150- if not self.urgency_map.has_key(raw_urgency):
151+ if raw_urgency not in self.urgency_map:
152 yield UploadWarning(
153 "Unable to grok urgency %s, overriding with 'low'"
154 % (raw_urgency))
155@@ -362,3 +350,26 @@
156 return self.changes_comment + changes_author
157
158
159+def determine_file_class_and_name(filename):
160+ """Determine the name and PackageUploadFile subclass for the filename."""
161+ source_match = re_issource.match(filename)
162+ binary_match = re_isadeb.match(filename)
163+ if source_match:
164+ package = source_match.group(1)
165+ if (determine_source_file_type(filename) ==
166+ SourcePackageFileType.DSC):
167+ cls = DSCFile
168+ else:
169+ cls = SourceUploadFile
170+ elif binary_match:
171+ package = binary_match.group(1)
172+ cls = {
173+ BinaryPackageFileType.DEB: DebBinaryUploadFile,
174+ BinaryPackageFileType.DDEB: DdebBinaryUploadFile,
175+ BinaryPackageFileType.UDEB: UdebBinaryUploadFile,
176+ }[determine_binary_file_type(filename)]
177+ else:
178+ raise CannotDetermineFileTypeError(
179+ "Could not determine the type of %r" % filename)
180+
181+ return package, cls
182
183=== added file 'lib/lp/archiveuploader/tests/test_changesfile.py'
184--- lib/lp/archiveuploader/tests/test_changesfile.py 1970-01-01 00:00:00 +0000
185+++ lib/lp/archiveuploader/tests/test_changesfile.py 2010-08-01 07:06:04 +0000
186@@ -0,0 +1,54 @@
187+# Copyright 2010 Canonical Ltd. This software is licensed under the
188+# GNU Affero General Public License version 3 (see the file LICENSE).
189+
190+"""Test ChangesFile functionality."""
191+
192+__metaclass__ = type
193+
194+from testtools import TestCase
195+
196+from lp.archiveuploader.changesfile import (CannotDetermineFileTypeError,
197+ determine_file_class_and_name)
198+from lp.archiveuploader.dscfile import DSCFile
199+from lp.archiveuploader.nascentuploadfile import (
200+ DebBinaryUploadFile, DdebBinaryUploadFile, SourceUploadFile,
201+ UdebBinaryUploadFile)
202+
203+
204+class TestDetermineFileClassAndName(TestCase):
205+
206+ def testSourceFile(self):
207+ # A non-DSC source file is a SourceUploadFile.
208+ self.assertEquals(
209+ ('foo', SourceUploadFile),
210+ determine_file_class_and_name('foo_1.0.diff.gz'))
211+
212+ def testDSCFile(self):
213+ # A DSC is a DSCFile, since they're special.
214+ self.assertEquals(
215+ ('foo', DSCFile),
216+ determine_file_class_and_name('foo_1.0.dsc'))
217+
218+ def testDEBFile(self):
219+ # A binary file is the appropriate PackageUploadFile subclass.
220+ self.assertEquals(
221+ ('foo', DebBinaryUploadFile),
222+ determine_file_class_and_name('foo_1.0_all.deb'))
223+ self.assertEquals(
224+ ('foo', DdebBinaryUploadFile),
225+ determine_file_class_and_name('foo_1.0_all.ddeb'))
226+ self.assertEquals(
227+ ('foo', UdebBinaryUploadFile),
228+ determine_file_class_and_name('foo_1.0_all.udeb'))
229+
230+ def testUnmatchingFile(self):
231+ # Files with unknown extensions or none at all are not
232+ # identified.
233+ self.assertRaises(
234+ CannotDetermineFileTypeError,
235+ determine_file_class_and_name,
236+ 'foo_1.0.notdsc')
237+ self.assertRaises(
238+ CannotDetermineFileTypeError,
239+ determine_file_class_and_name,
240+ 'foo')
241
242=== modified file 'lib/lp/archiveuploader/tests/test_utils.py'
243--- lib/lp/archiveuploader/tests/test_utils.py 2010-07-24 05:41:31 +0000
244+++ lib/lp/archiveuploader/tests/test_utils.py 2010-08-01 07:06:04 +0000
245@@ -75,6 +75,11 @@
246 determine_binary_file_type('foo_1.0-1_all.deb'),
247 BinaryPackageFileType.DEB)
248
249+ # .ddeb -> DDEB
250+ self.assertEquals(
251+ determine_binary_file_type('foo_1.0-1_all.ddeb'),
252+ BinaryPackageFileType.DDEB)
253+
254 # .udeb -> UDEB
255 self.assertEquals(
256 determine_binary_file_type('foo_1.0-1_all.udeb'),
257
258=== modified file 'lib/lp/archiveuploader/utils.py'
259--- lib/lp/archiveuploader/utils.py 2009-12-10 13:53:30 +0000
260+++ lib/lp/archiveuploader/utils.py 2010-08-01 07:06:04 +0000
261@@ -98,6 +98,8 @@
262 return BinaryPackageFileType.DEB
263 elif filename.endswith(".udeb"):
264 return BinaryPackageFileType.UDEB
265+ elif filename.endswith(".ddeb"):
266+ return BinaryPackageFileType.DDEB
267 else:
268 return None
269
270@@ -128,7 +130,7 @@
271 return (section, component)
272
273
274-def build_file_list(tagfile, is_dsc = False, default_component="main" ):
275+def build_file_list(tagfile, is_dsc = False, default_component="main"):
276 files = {}
277
278 if "files" not in tagfile:
279@@ -170,7 +172,7 @@
280 "size": size,
281 "section": section,
282 "priority": priority,
283- "component": component
284+ "component": component,
285 }
286
287 return files
288@@ -185,7 +187,7 @@
289 unicode(s, 'utf-8')
290 return s
291 except UnicodeError:
292- latin1_s = unicode(s,'iso8859-1')
293+ latin1_s = unicode(s, 'iso8859-1')
294 return latin1_s.encode('utf-8')
295
296
297@@ -221,11 +223,11 @@
298
299 def __init__(self, message):
300 Exception.__init__(self)
301- self.args = (message,)
302+ self.args = (message, )
303 self.message = message
304
305
306-def fix_maintainer (maintainer, field_name="Maintainer"):
307+def fix_maintainer(maintainer, field_name="Maintainer"):
308 """Parses a Maintainer or Changed-By field and returns:
309
310 (1) an RFC822 compatible version,
311@@ -290,4 +292,3 @@
312 content = ascii_smash(content)
313
314 return fix_maintainer(content, fieldname)
315-