Launchpad itself

Merge lp:~wgrant/launchpad/refactor-nuf-creation into lp:launchpad

  1. refactor-nuf-creation
  2. Merge into devel
Proposed by William Grant
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11275
Proposed branch: lp:~wgrant/launchpad/refactor-nuf-creation
Merge into: lp:launchpad
Diff against target: 315 lines (+124/-53)
4 files modified
lib/lp/archiveuploader/changesfile.py (+58/-47)
lib/lp/archiveuploader/tests/test_changesfile.py (+54/-0)
lib/lp/archiveuploader/tests/test_utils.py (+5/-0)
lib/lp/archiveuploader/utils.py (+7/-6)
To merge this branch: bzr merge lp:~wgrant/launchpad/refactor-nuf-creation
Reviewer Review Type Date Requested Status
Jelmer Vernooij (community) code Approve
Review via email: mp+30851@code.launchpad.net

Commit message

Factor out and test part of ChangesFile.processFiles, so we can use it later in NascentUpload tests.

Description of the change

This branch factors out and tests part of ChangesFile.processFiles. This will be used by a later branch to enable NascentUpload tests without having to create real uploads.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) wrote :

Nice work.

Rather than returning None it seems more sensible to me to raise some sort of exception. With that change, r=me.

review: Needs Fixing (code)
Revision history for this message
Jelmer Vernooij (jelmer) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/archiveuploader/changesfile.py'
--- lib/lp/archiveuploader/changesfile.py 2009-10-12 09:55:31 +0000
+++ lib/lp/archiveuploader/changesfile.py 2010-08-01 07:06:04 +0000
@@ -10,21 +10,30 @@
10__metaclass__ = type10__metaclass__ = type
1111
12__all__ = [12__all__ = [
13 'ChangesFile'13 'CannotDetermineFileTypeError',
14 'ChangesFile',
15 'determine_file_class_and_name',
14 ]16 ]
1517
16import os18import os
1719
18from lp.archiveuploader.dscfile import DSCFile, SignableTagFile20from lp.archiveuploader.dscfile import DSCFile, SignableTagFile
19from lp.archiveuploader.nascentuploadfile import (21from lp.archiveuploader.nascentuploadfile import (
20 BaseBinaryUploadFile, CustomUploadFile, DdebBinaryUploadFile,22 BaseBinaryUploadFile, CustomUploadFile, DdebBinaryUploadFile,
21 DebBinaryUploadFile, SourceUploadFile, UdebBinaryUploadFile,23 DebBinaryUploadFile, SourceUploadFile, UdebBinaryUploadFile,
22 UploadError, UploadWarning, splitComponentAndSection)24 UploadError, UploadWarning, splitComponentAndSection)
23from lp.archiveuploader.utils import (25from lp.archiveuploader.utils import (
26 determine_binary_file_type, determine_source_file_type,
24 re_isadeb, re_issource, re_changes_file_name)27 re_isadeb, re_issource, re_changes_file_name)
25from lp.archiveuploader.tagfiles import (28from lp.archiveuploader.tagfiles import (
26 parse_tagfile, TagFileParseError)29 parse_tagfile, TagFileParseError)
27from canonical.launchpad.interfaces import SourcePackageUrgency30from lp.registry.interfaces.sourcepackage import (SourcePackageFileType,
31 SourcePackageUrgency)
32from lp.soyuz.interfaces.binarypackagerelease import BinaryPackageFileType
33
34
35class CannotDetermineFileTypeError(Exception):
36 """The type of the given file could not be determined."""
2837
2938
30class ChangesFile(SignableTagFile):39class ChangesFile(SignableTagFile):
@@ -42,7 +51,7 @@
42 "medium": SourcePackageUrgency.MEDIUM,51 "medium": SourcePackageUrgency.MEDIUM,
43 "high": SourcePackageUrgency.HIGH,52 "high": SourcePackageUrgency.HIGH,
44 "critical": SourcePackageUrgency.EMERGENCY,53 "critical": SourcePackageUrgency.EMERGENCY,
45 "emergency": SourcePackageUrgency.EMERGENCY54 "emergency": SourcePackageUrgency.EMERGENCY,
46 }55 }
4756
48 dsc = None57 dsc = None
@@ -103,7 +112,7 @@
103112
104 def checkFileName(self):113 def checkFileName(self):
105 """Make sure the changes file name is well-formed.114 """Make sure the changes file name is well-formed.
106 115
107 Please note: for well-formed changes file names the `filename_archtag`116 Please note: for well-formed changes file names the `filename_archtag`
108 property will be set appropriately.117 property will be set appropriately.
109 """118 """
@@ -152,8 +161,9 @@
152 def processFiles(self):161 def processFiles(self):
153 """Build objects for each file mentioned in this changesfile.162 """Build objects for each file mentioned in this changesfile.
154163
155 This method is an error generator, i.e, it returns an iterator over all164 This method is an error generator, i.e, it returns an iterator over
156 exceptions that are generated while processing all mentioned files.165 all exceptions that are generated while processing all mentioned
166 files.
157 """167 """
158 files = []168 files = []
159 for fileline in self._dict['files'].strip().split("\n"):169 for fileline in self._dict['files'].strip().split("\n"):
@@ -161,8 +171,6 @@
161 # CHECKSUM SIZE [COMPONENT/]SECTION PRIORITY FILENAME171 # CHECKSUM SIZE [COMPONENT/]SECTION PRIORITY FILENAME
162 digest, size, component_and_section, priority_name, filename = (172 digest, size, component_and_section, priority_name, filename = (
163 fileline.strip().split())173 fileline.strip().split())
164 source_match = re_issource.match(filename)
165 binary_match = re_isadeb.match(filename)
166 filepath = os.path.join(self.dirname, filename)174 filepath = os.path.join(self.dirname, filename)
167 try:175 try:
168 if self.isCustom(component_and_section):176 if self.isCustom(component_and_section):
@@ -172,42 +180,22 @@
172 file_instance = CustomUploadFile(180 file_instance = CustomUploadFile(
173 filepath, digest, size, component_and_section,181 filepath, digest, size, component_and_section,
174 priority_name, self.policy, self.logger)182 priority_name, self.policy, self.logger)
175 elif source_match:183 else:
176 package = source_match.group(1)184 try:
177 if filename.endswith("dsc"):185 package, cls = determine_file_class_and_name(filename)
178 file_instance = DSCFile(186 except CannotDetermineFileTypeError:
179 filepath, digest, size, component_and_section,187 yield UploadError(
180 priority_name, package, self.version, self,188 "Unable to identify file %s (%s) in changes."
181 self.policy, self.logger)189 % (filename, component_and_section))
182 # Store the DSC because it is very convenient190 continue
191
192 file_instance = cls(
193 filepath, digest, size, component_and_section,
194 priority_name, package, self.version, self,
195 self.policy, self.logger)
196
197 if cls == DSCFile:
183 self.dsc = file_instance198 self.dsc = file_instance
184 else:
185 file_instance = SourceUploadFile(
186 filepath, digest, size, component_and_section,
187 priority_name, package, self.version, self,
188 self.policy, self.logger)
189 elif binary_match:
190 package = binary_match.group(1)
191 if filename.endswith("udeb"):
192 file_instance = UdebBinaryUploadFile(
193 filepath, digest, size, component_and_section,
194 priority_name, package, self.version, self,
195 self.policy, self.logger)
196 elif filename.endswith("ddeb"):
197 file_instance = DdebBinaryUploadFile(
198 filepath, digest, size, component_and_section,
199 priority_name, package, self.version, self,
200 self.policy, self.logger)
201 else:
202 file_instance = DebBinaryUploadFile(
203 filepath, digest, size, component_and_section,
204 priority_name, package, self.version, self,
205 self.policy, self.logger)
206 else:
207 yield UploadError(
208 "Unable to identify file %s (%s) in changes."
209 % (filename, component_and_section))
210 continue
211 except UploadError, error:199 except UploadError, error:
212 yield error200 yield error
213 else:201 else:
@@ -218,8 +206,8 @@
218 def verify(self):206 def verify(self):
219 """Run all the verification checks on the changes data.207 """Run all the verification checks on the changes data.
220208
221 This method is an error generator, i.e, it returns an iterator over all209 This method is an error generator, i.e, it returns an iterator over
222 exceptions that are generated while verifying the changesfile210 all exceptions that are generated while verifying the changesfile
223 consistency.211 consistency.
224 """212 """
225 self.logger.debug("Verifying the changes file.")213 self.logger.debug("Verifying the changes file.")
@@ -228,7 +216,7 @@
228 yield UploadError("No files found in the changes")216 yield UploadError("No files found in the changes")
229217
230 raw_urgency = self._dict['urgency'].lower()218 raw_urgency = self._dict['urgency'].lower()
231 if not self.urgency_map.has_key(raw_urgency):219 if raw_urgency not in self.urgency_map:
232 yield UploadWarning(220 yield UploadWarning(
233 "Unable to grok urgency %s, overriding with 'low'"221 "Unable to grok urgency %s, overriding with 'low'"
234 % (raw_urgency))222 % (raw_urgency))
@@ -362,3 +350,26 @@
362 return self.changes_comment + changes_author350 return self.changes_comment + changes_author
363351
364352
353def determine_file_class_and_name(filename):
354 """Determine the name and PackageUploadFile subclass for the filename."""
355 source_match = re_issource.match(filename)
356 binary_match = re_isadeb.match(filename)
357 if source_match:
358 package = source_match.group(1)
359 if (determine_source_file_type(filename) ==
360 SourcePackageFileType.DSC):
361 cls = DSCFile
362 else:
363 cls = SourceUploadFile
364 elif binary_match:
365 package = binary_match.group(1)
366 cls = {
367 BinaryPackageFileType.DEB: DebBinaryUploadFile,
368 BinaryPackageFileType.DDEB: DdebBinaryUploadFile,
369 BinaryPackageFileType.UDEB: UdebBinaryUploadFile,
370 }[determine_binary_file_type(filename)]
371 else:
372 raise CannotDetermineFileTypeError(
373 "Could not determine the type of %r" % filename)
374
375 return package, cls
365376
=== added file 'lib/lp/archiveuploader/tests/test_changesfile.py'
--- lib/lp/archiveuploader/tests/test_changesfile.py 1970-01-01 00:00:00 +0000
+++ lib/lp/archiveuploader/tests/test_changesfile.py 2010-08-01 07:06:04 +0000
@@ -0,0 +1,54 @@
1# Copyright 2010 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Test ChangesFile functionality."""
5
6__metaclass__ = type
7
8from testtools import TestCase
9
10from lp.archiveuploader.changesfile import (CannotDetermineFileTypeError,
11 determine_file_class_and_name)
12from lp.archiveuploader.dscfile import DSCFile
13from lp.archiveuploader.nascentuploadfile import (
14 DebBinaryUploadFile, DdebBinaryUploadFile, SourceUploadFile,
15 UdebBinaryUploadFile)
16
17
18class TestDetermineFileClassAndName(TestCase):
19
20 def testSourceFile(self):
21 # A non-DSC source file is a SourceUploadFile.
22 self.assertEquals(
23 ('foo', SourceUploadFile),
24 determine_file_class_and_name('foo_1.0.diff.gz'))
25
26 def testDSCFile(self):
27 # A DSC is a DSCFile, since they're special.
28 self.assertEquals(
29 ('foo', DSCFile),
30 determine_file_class_and_name('foo_1.0.dsc'))
31
32 def testDEBFile(self):
33 # A binary file is the appropriate PackageUploadFile subclass.
34 self.assertEquals(
35 ('foo', DebBinaryUploadFile),
36 determine_file_class_and_name('foo_1.0_all.deb'))
37 self.assertEquals(
38 ('foo', DdebBinaryUploadFile),
39 determine_file_class_and_name('foo_1.0_all.ddeb'))
40 self.assertEquals(
41 ('foo', UdebBinaryUploadFile),
42 determine_file_class_and_name('foo_1.0_all.udeb'))
43
44 def testUnmatchingFile(self):
45 # Files with unknown extensions or none at all are not
46 # identified.
47 self.assertRaises(
48 CannotDetermineFileTypeError,
49 determine_file_class_and_name,
50 'foo_1.0.notdsc')
51 self.assertRaises(
52 CannotDetermineFileTypeError,
53 determine_file_class_and_name,
54 'foo')
055
=== modified file 'lib/lp/archiveuploader/tests/test_utils.py'
--- lib/lp/archiveuploader/tests/test_utils.py 2010-07-24 05:41:31 +0000
+++ lib/lp/archiveuploader/tests/test_utils.py 2010-08-01 07:06:04 +0000
@@ -75,6 +75,11 @@
75 determine_binary_file_type('foo_1.0-1_all.deb'),75 determine_binary_file_type('foo_1.0-1_all.deb'),
76 BinaryPackageFileType.DEB)76 BinaryPackageFileType.DEB)
7777
78 # .ddeb -> DDEB
79 self.assertEquals(
80 determine_binary_file_type('foo_1.0-1_all.ddeb'),
81 BinaryPackageFileType.DDEB)
82
78 # .udeb -> UDEB83 # .udeb -> UDEB
79 self.assertEquals(84 self.assertEquals(
80 determine_binary_file_type('foo_1.0-1_all.udeb'),85 determine_binary_file_type('foo_1.0-1_all.udeb'),
8186
=== modified file 'lib/lp/archiveuploader/utils.py'
--- lib/lp/archiveuploader/utils.py 2009-12-10 13:53:30 +0000
+++ lib/lp/archiveuploader/utils.py 2010-08-01 07:06:04 +0000
@@ -98,6 +98,8 @@
98 return BinaryPackageFileType.DEB98 return BinaryPackageFileType.DEB
99 elif filename.endswith(".udeb"):99 elif filename.endswith(".udeb"):
100 return BinaryPackageFileType.UDEB100 return BinaryPackageFileType.UDEB
101 elif filename.endswith(".ddeb"):
102 return BinaryPackageFileType.DDEB
101 else:103 else:
102 return None104 return None
103105
@@ -128,7 +130,7 @@
128 return (section, component)130 return (section, component)
129131
130132
131def build_file_list(tagfile, is_dsc = False, default_component="main" ):133def build_file_list(tagfile, is_dsc = False, default_component="main"):
132 files = {}134 files = {}
133135
134 if "files" not in tagfile:136 if "files" not in tagfile:
@@ -170,7 +172,7 @@
170 "size": size,172 "size": size,
171 "section": section,173 "section": section,
172 "priority": priority,174 "priority": priority,
173 "component": component175 "component": component,
174 }176 }
175177
176 return files178 return files
@@ -185,7 +187,7 @@
185 unicode(s, 'utf-8')187 unicode(s, 'utf-8')
186 return s188 return s
187 except UnicodeError:189 except UnicodeError:
188 latin1_s = unicode(s,'iso8859-1')190 latin1_s = unicode(s, 'iso8859-1')
189 return latin1_s.encode('utf-8')191 return latin1_s.encode('utf-8')
190192
191193
@@ -221,11 +223,11 @@
221223
222 def __init__(self, message):224 def __init__(self, message):
223 Exception.__init__(self)225 Exception.__init__(self)
224 self.args = (message,)226 self.args = (message, )
225 self.message = message227 self.message = message
226228
227229
228def fix_maintainer (maintainer, field_name="Maintainer"):230def fix_maintainer(maintainer, field_name="Maintainer"):
229 """Parses a Maintainer or Changed-By field and returns:231 """Parses a Maintainer or Changed-By field and returns:
230232
231 (1) an RFC822 compatible version,233 (1) an RFC822 compatible version,
@@ -290,4 +292,3 @@
290 content = ascii_smash(content)292 content = ascii_smash(content)
291293
292 return fix_maintainer(content, fieldname)294 return fix_maintainer(content, fieldname)
293