Launchpad itself

Merge lp:~jelmer/launchpad/publisher-use-debian-1 into lp:launchpad

  1. publisher-use-debian-1
  2. Merge into devel
Proposed by Jelmer Vernooij
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11404
Proposed branch: lp:~jelmer/launchpad/publisher-use-debian-1
Merge into: lp:launchpad
Diff against target: 211 lines (+62/-68)
2 files modified
lib/lp/archivepublisher/publishing.py (+61/-67)
lib/lp/soyuz/doc/soyuz-upload.txt (+1/-1)
To merge this branch: bzr merge lp:~jelmer/launchpad/publisher-use-debian-1
Reviewer Review Type Date Requested Status
Brad Crittenden (community) code Approve
Review via email: mp+32245@code.launchpad.net

Commit message

Use python-debian to generate Release files in the archivepublisher.

Description of the change

Refactor lp.archivepublisher.publishing to use the functionality in the new "debian" Python module to generate Release files, rather than using hardcoded strings.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) wrote :

I haven't done a pre-implementation call about this specific change, although I have discussed moving towards more use of python-debian (which is an upstream project maintained by various Debian and Ubuntu developers) with Julian in the past.

I'm about to make some more changes to this particular part of the code, and it seemed like a good idea to refactor first and use python-debian's infrastructure so that I don't have to worry about the serialization of newly introduced fields.

Tests: ./bin/test lp.archivepublisher

I've fixed all remaining lint issues.

Revision history for this message
Brad Crittenden (bac) wrote :

Very nice branch Jelmer. Please use a more descriptive loop variable than "f", especially since it is used again a little later as a file descriptor. Recall our coding guidelines strongly discourage single letter variables, even in loops.

review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archivepublisher/publishing.py'
2--- lib/lp/archivepublisher/publishing.py 2010-07-07 06:28:03 +0000
3+++ lib/lp/archivepublisher/publishing.py 2010-08-20 11:35:17 +0000
4@@ -14,6 +14,8 @@
5 import os
6 import shutil
7
8+from debian.deb822 import Release
9+
10 from datetime import datetime
11
12 from zope.component import getUtility
13@@ -40,24 +42,6 @@
14
15 suffixpocket = dict((v, k) for (k, v) in pocketsuffix.items())
16
17-DISTRORELEASE_STANZA = """Origin: %s
18-Label: %s
19-Suite: %s
20-Version: %s
21-Codename: %s
22-Date: %s
23-Architectures: %s
24-Components: %s
25-Description: %s
26-"""
27-
28-DISTROARCHRELEASE_STANZA = """Archive: %s
29-Version: %s
30-Component: %s
31-Origin: %s
32-Label: %s
33-Architecture: %s
34-"""
35
36 def reorder_components(components):
37 """Return a list of the components provided.
38@@ -92,6 +76,7 @@
39
40 return dp
41
42+
43 def getPublisher(archive, allowed_suites, log, distsroot=None):
44 """Return an initialised Publisher instance for the given context.
45
46@@ -487,33 +472,43 @@
47 else:
48 drsummary += pocket.name.capitalize()
49
50+ release_file = Release()
51+ release_file["Origin"] = self._getOrigin()
52+ release_file["Label"] = self._getLabel()
53+ release_file["Suite"] = full_name
54+ release_file["Version"] = distroseries.version
55+ release_file["Codename"] = distroseries.name
56+ release_file["Date"] = datetime.utcnow().strftime(
57+ "%a, %d %b %Y %k:%M:%S UTC")
58+ release_file["Architectures"] = " ".join(
59+ sorted(list(all_architectures)))
60+ release_file["Components"] = " ".join(
61+ reorder_components(all_components))
62+ release_file["Description"] = drsummary
63+
64+ for filename in sorted(list(all_files), key=os.path.dirname):
65+ entry = self._readIndexFileContents(full_name, filename)
66+ if entry is None:
67+ continue
68+ release_file.setdefault("MD5Sum", []).append({
69+ "md5sum": hashlib.md5(entry).hexdigest(),
70+ "name": filename,
71+ "size": len(entry)})
72+ release_file.setdefault("SHA1", []).append({
73+ "sha1": hashlib.sha1(entry).hexdigest(),
74+ "name": filename,
75+ "size": len(entry)})
76+ release_file.setdefault("SHA256", []).append({
77+ "sha256": hashlib.sha256(entry).hexdigest(),
78+ "name": filename,
79+ "size": len(entry)})
80+
81 f = open(os.path.join(
82 self._config.distsroot, full_name, "Release"), "w")
83-
84- stanza = (DISTRORELEASE_STANZA % (
85- self._getOrigin(),
86- self._getLabel(),
87- full_name,
88- distroseries.version,
89- distroseries.name,
90- datetime.utcnow().strftime("%a, %d %b %Y %k:%M:%S UTC"),
91- " ".join(sorted(list(all_architectures))),
92- " ".join(reorder_components(all_components)),
93- drsummary)).encode("utf-8")
94- f.write(stanza)
95-
96- f.write("MD5Sum:\n")
97- all_files = sorted(list(all_files), key=os.path.dirname)
98- for file_name in all_files:
99- self._writeSumLine(full_name, f, file_name, hashlib.md5)
100- f.write("SHA1:\n")
101- for file_name in all_files:
102- self._writeSumLine(full_name, f, file_name, hashlib.sha1)
103- f.write("SHA256:\n")
104- for file_name in all_files:
105- self._writeSumLine(full_name, f, file_name, hashlib.sha256)
106-
107- f.close()
108+ try:
109+ release_file.dump(f, "utf-8")
110+ finally:
111+ f.close()
112
113 # Skip signature if the archive signing key is undefined.
114 if self.archive.signing_key is None:
115@@ -562,26 +557,29 @@
116
117 all_files.add(os.path.join(component, architecture, "Release"))
118
119+ release_file = Release()
120+ release_file["Archive"] = full_name
121+ release_file["Version"] = distroseries.version
122+ release_file["Component"] = component
123+ release_file["Origin"] = self._getOrigin()
124+ release_file["Label"] = self._getLabel()
125+ release_file["Architecture"] = clean_architecture
126+
127 f = open(os.path.join(self._config.distsroot, full_name,
128 component, architecture, "Release"), "w")
129-
130- stanza = (DISTROARCHRELEASE_STANZA % (
131- full_name,
132- distroseries.version,
133- component,
134- self._getOrigin(),
135- self._getLabel(),
136- unicode(clean_architecture))).encode("utf-8")
137- f.write(stanza)
138- f.close()
139+ try:
140+ release_file.dump(f, "utf-8")
141+ finally:
142+ f.close()
143
144 return clean_architecture
145
146- def _writeSumLine(self, distroseries_name, out_file, file_name, sum_form):
147- """Write out a checksum line.
148+ def _readIndexFileContents(self, distroseries_name, file_name):
149+ """Read an index files' contents.
150
151- Writes a checksum to the given file for the given filename in
152- the given form.
153+ :param distroseries_name: Distro series name
154+ :param file_name: Filename relative to the parent container directory.
155+ :return: File contents, or None if the file could not be found.
156 """
157 full_name = os.path.join(self._config.distsroot,
158 distroseries_name, file_name)
159@@ -590,25 +588,21 @@
160 # Most likely we have an incomplete archive (E.g. no sources
161 # for a given distroseries). This is a non-fatal issue
162 self.log.debug("Failed to find " + full_name)
163- return
164+ return None
165
166 in_file = open(full_name, 'r')
167 try:
168- contents = in_file.read()
169- length = len(contents)
170- checksum = sum_form(contents).hexdigest()
171+ return in_file.read()
172 finally:
173 in_file.close()
174
175- out_file.write(" %s % 16d %s\n" % (checksum, length, file_name))
176-
177 def deleteArchive(self):
178 """Delete the archive.
179-
180- Physically remove the entire archive from disk and set the archive's
181+
182+ Physically remove the entire archive from disk and set the archive's
183 status to DELETED.
184
185- Any errors encountered while removing the archive from disk will
186+ Any errors encountered while removing the archive from disk will
187 be caught and an OOPS report generated.
188 """
189
190@@ -627,7 +621,7 @@
191 self.log.warning(
192 "Failed to delete directory '%s' for archive "
193 "'%s/%s'\n%s" % (
194- directory, self.archive.owner.name,
195+ directory, self.archive.owner.name,
196 self.archive.name, e))
197
198 self.archive.status = ArchiveStatus.DELETED
199
200=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
201--- lib/lp/soyuz/doc/soyuz-upload.txt 2010-06-29 14:01:01 +0000
202+++ lib/lp/soyuz/doc/soyuz-upload.txt 2010-08-20 11:35:17 +0000
203@@ -708,7 +708,7 @@
204 Version: 6.6.6
205 Codename: breezy-autotest
206 Date: ...
207- Architectures:
208+ Architectures:
209 Components: main restricted universe multiverse
210 Description: ubuntutest Breezy Badger Autotest 6.6.6
211 MD5Sum: