Launchpad itself

Merge lp:~kfogel/launchpad/add-community-contributions-script into lp:launchpad

  1. add-community-contributions-script
  2. Merge into devel
Proposed by Karl Fogel
Status: Merged
Merged at revision: not available
Proposed branch: lp:~kfogel/launchpad/add-community-contributions-script
Merge into: lp:launchpad
Diff against target: None lines
To merge this branch: bzr merge lp:~kfogel/launchpad/add-community-contributions-script
Reviewer Review Type Date Requested Status
Jonathan Lange Pending
Review via email: mp+11417@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Karl Fogel (kfogel) wrote :

Add a script that detects community contributions and updates a wiki page with information about them. This script doesn't affect Launchpad itself at all; it just takes a Launchpad branch as read-only input.

Note the dependency on editmoin.py. The import just errors informatively if editmoin can't be found, so the user will know where to get it. This seemed preferable to dragging editmoin.py into utilities/.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'utilities/community-contributions.py'
--- utilities/community-contributions.py 1970-01-01 00:00:00 +0000
+++ utilities/community-contributions.py 2009-09-09 04:39:12 +0000
@@ -0,0 +1,387 @@
1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3#
4# Copyright 2009 Canonical Ltd. This software is licensed under the
5# GNU Affero General Public License version 3 (see the file LICENSE).
6
7"""Show what Launchpad community contributors have done, by trawling
8a Launchpad branch's history, detecting contributions by non-Canonical
9developers, and updating https://dev.launchpad.net/Contributions accordingly.
10
11Usage: community-contributions.py [options] PATH_TO_LAUNCHPAD_DEVEL_BRANCH
12
13Requirements:
14 You need a 'devel' branch of Launchpad available locally (see
15 https://dev.launchpad.net/Getting), your ~/.moin_ids file must
16 be set up correctly, and you need editmoin.py (if you don't
17 have it, the error message will tell you where to get it).
18
19Options:
20 -q Print no non-essential messages.
21 -h, --help Print this help.
22 --dry-run Don't update the wiki, just print the new wiki page to stdout.
23"""
24
25# General notes:
26#
27# The Right Way to do this would probably be to output some kind of
28# XML format, and then have a separate converter script transform that
29# to wiki syntax and update the wiki page. But as the wiki is our
30# only consumer right now, we just output wiki syntax and update the
31# wiki page directly, premature generalization being the root of all
32# evil.
33#
34# For understanding the code, you may find it helpful to see
35# bzrlib/log.py and http://bazaar-vcs.org/Integrating_with_Bazaar.
36
37import re
38import sys
39import getopt
40from bzrlib.branch import Branch
41from bzrlib import log
42from bzrlib.osutils import format_date
43
44try:
45 from editmoin import editshortcut
46except:
47 sys.stderr.write("""ERROR: Unable to import from 'editmoin'. How to solve:
48As of 2009-09-01, you can get editmoin.py from
49
50 https://bazaar.launchpad.net/~kfogel/lp-dev-utils/lp-user-tools/files
51
52(This is a transitional location; it may move to a more public place.)
53""")
54 sys.exit(1)
55
56
57# While anyone with "@canonical.com" in their email address will be
58# counted as a Canonical contributor, sometimes Canonical people
59# submit from personal addresses, so we still need a list.
60#
61# ### TODO: Really, this ought to use launchpadlib to consult
62# ### Launchpad itself to find out who's a Canonical developer.
63known_canonical_devs = (
64 u'Aaron Bentley',
65 u'Abel Deuring',
66 u'Adam Conrad',
67 u'Andrew Bennetts',
68 u'Barry Warsaw',
69 u'Brad Crittenden',
70 u'Carlos Perello Marin',
71 u'Carlos Perelló Marín',
72 u'Celso Providelo',
73 u'Christian Robottom Reis',
74 u'Cody Somerville',
75 u'Curtis Hovey',
76 u'Dafydd Harries',
77 u'Daniel Silverstone',
78 u'Danilo Šegan',
79 u'David Allouche',
80 u'Deryck Hodge',
81 u'Diogo Matsubara',
82 u'Elliot Murphy',
83 u'Francis J. Lacoste',
84 u'Gabriel Neuman gneuman@async.com',
85 u'Gary Poster',
86 u'Guilherme Salgado',
87 u'Gustavo Niemeyer',
88 u'Henning Eggers',
89 u'Herb McNew',
90 u'James Henstridge',
91 u'Jeroen Vermeulen',
92 u'Jonathan Knowles',
93 u'Jonathan Lange',
94 u'Julian Edwards',
95 u'Karl Fogel',
96 u'Launch Pad',
97 u'Launchpad Developers',
98 u'Leonard Richardson',
99 u'Malcolm Cleaton',
100 u'Maris Fogels',
101 u'Martin Albisetti',
102 u'Matt Zimmerman',
103 u'Matthew Revell',
104 u'Michael Hudson',
105 u'Michael Nelson',
106 u'Muharem Hrnjadovic',
107 u'Patch Queue Manager',
108 u'Paul Hummer',
109 u'Robert Collins',
110 u'Sidnei',
111 u'Sidnei da Silva',
112 u'Steve McInerney',
113 u'Stuart Bishop',
114 u'Tom Berger',
115 u'david',
116 u'jml@mumak.net',
117 u'kiko@beetle',
118 )
119
120
121class RevisionError(Exception):
122 pass;
123
124
125class ContainerRevision():
126 """A wrapper for a top-level LogRevision containing child LogRevisions."""
127
128 def __init__(self, top_lr):
129 self.top_rev = top_lr # e.g. LogRevision for r9371.
130 self.contained_revs = [ ] # e.g. [ {9369.1.1}, {9206.4.4}, ... ],
131 # where "{X}" means "LogRevision for X"
132 def add_subrev(self, lr):
133 """Add a descendant child of this container revision."""
134 self.contained_revs.append(lr)
135
136 def __str__(self):
137 timestamp = self.top_rev.rev.timestamp
138 timezone = self.top_rev.rev.timezone
139 message = self.top_rev.rev.message or "(NO LOG MESSAGE)"
140 rev_id = self.top_rev.rev.revision_id or "(NO REVISION ID)"
141 inventory_sha1 = self.top_rev.rev.inventory_sha1
142 if timestamp:
143 date_str = format_date(timestamp, timezone or 0, 'original')
144 else:
145 date_str = "(NO DATE)"
146
147 # ### TODO: just using 'devel' branch for now. We have four
148 # ### trunks; that makes life hard. Not sure what to do about
149 # ### that; unifying the data is possible, but a bit of work.
150 # ### See https://dev.launchpad.net/Trunk for more information.
151 rev_url_base = "http://bazaar.launchpad.net/~launchpad-pqm/" \
152 "launchpad/devel/revision/"
153
154 # In loggerhead, you can use either a revision number or a
155 # revision ID. In other words, these would reach the same page:
156 #
157 # http://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/revision/9202
158 #
159 # -and-
160 #
161 # http://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/revision/\
162 # launchpad@pqm.canonical.com-20090821221206-ritpv21q8w61gbpt
163 #
164 # In our links, even when the link text is a revnum, we still
165 # use a rev-id for the target. This is both so that the URL will
166 # still work if you manually tweak it (say from "devel" to
167 # "devel") and so that hovering over a revnum on the wiki page
168 # will give you some information about it before you click
169 # (because a rev id often identifies the committer).
170 rev_id_url = rev_url_base + rev_id
171 s = " * [[%s|r%s]] -- %s\n" % (rev_id_url, self.top_rev.revno, date_str)
172 s += " {{{\n%s\n}}}\n" % message
173 s += " '''Commits:'''\n "
174 s += "\n ".join(["[[%s|%s]]" % (rev_url_base + lr.rev.revision_id,
175 lr.revno) for lr in self.contained_revs])
176 s += "\n"
177 return s
178
179
180# "ExternalContributor" is too much to type, so I guess we'll just use this.
181class ExCon():
182 """A contributor to Launchpad from outside Canonical."""
183 def __init__(self, name):
184 """Create a new external contributor named NAME. NAME is usually
185 e.g. "Veronica Random <veronica@example.com>", but any "@"-sign
186 will be disguised in the new object."""
187
188 self.name = name.replace("@", " {_AT_} ")
189 # If name is "Veronica Random <veronica {_AT_} example.com>",
190 # then name_as_anchor will be "veronica_random".
191 self.name_as_anchor = \
192 re.compile("\\s+").sub("_", name.split("<")[0].strip()).lower()
193 # All the top-level revisions this contributor is associated with
194 # (key == value == ContainerRevision). We use a dictionary
195 # instead of list to get set semantics; set() would be overkill.
196 self._landings = { }
197
198 def num_landings(self):
199 """Return the number of top-level landings that include revisions
200 by this contributor."""
201 return len(self._landings)
202
203 def add_top_level_revision(self, cr):
204 """Record ContainableRevision CR as associated with this contributor."""
205 self._landings[cr] = cr
206
207 def show_contributions(self):
208 """Return a wikified string showing this contributor's contributions."""
209 s = "== %s ==\n\n" % self.name
210 plural = "s"
211 if self.num_landings() == 1:
212 plural = ""
213 s += "''%d top-level landing%s:''\n\n" % (self.num_landings(), plural)
214 def prefer_recent_revs(a, b):
215 # A and B are LogRevisions; put more recent ones higher in the list.
216 return cmp(b.top_rev.revno, a.top_rev.revno)
217 for cr in sorted(self._landings, prefer_recent_revs):
218 s += str(cr)
219 s += "\n"
220 return s
221
222
223def get_ex_cons(authors, all_ex_cons):
224 """Return a list of ExCon objects corresponding to AUTHORS (a list
225 of strings). If there are no external contributors in authors,
226 return an empty list.
227
228 ALL_EX_CONS is a dictionary mapping author names (as received from
229 the bzr logs, i.e., with email address undisguised) to ExCon objects.
230 """
231 ex_cons_this_rev = [ ]
232 for a in authors:
233 known = False
234 for name_fragment in known_canonical_devs:
235 if u"@canonical.com" in a or name_fragment in a:
236 known = True
237 break
238 if not known:
239 ### There's a variant of the Singleton pattern that could be
240 ### used for this, whereby instantiating an ExCon object would
241 ### just get back an existing object if such has already been
242 ### instantiated for this name. But that would make this code
243 ### non-reentrant, and that's just not cool.
244 if all_ex_cons.has_key(a):
245 ec = all_ex_cons[a]
246 else:
247 ec = ExCon(a)
248 all_ex_cons[a] = ec
249 ex_cons_this_rev.append(ec)
250 return ex_cons_this_rev
251
252
253# The LogFormatter abstract class should really be called LogReceiver
254# or something -- subclasses don't have to be about display.
255class LogExCons(log.LogFormatter):
256 """Log all the external contributions, by Contributor."""
257 # See log.LogFormatter documentation.
258 supports_merge_revisions = True
259
260 def __init__(self):
261 super(LogExCons, self).__init__(to_file=None)
262 # Dictionary mapping author names (with undisguised email
263 # addresses) to ExCon objects.
264 self.all_ex_cons = { }
265 # ContainerRevision object representing most-recently-seen top-level rev.
266 current_top_level_rev = None
267
268 def result(self):
269 """Return a moin-wiki-syntax string with TOC followed by contributions."""
270 def prefer_more_revs(a, b):
271 # List the most prolific contributors first.
272 return cmp(b.num_landings(), a.num_landings())
273 sorted_contributors = \
274 sorted(self.all_ex_cons.values(), prefer_more_revs)
275 s = "-----\n\n"
276 s += "= Who =\n\n"
277 for val in sorted_contributors:
278 plural = "s"
279 if val.num_landings() == 1:
280 plural = ""
281 s += " 1. [[#%s|%s]] ''(%d top-level landing%s)''\n" \
282 % (val.name_as_anchor, val.name, val.num_landings(), plural)
283 s += "\n-----\n\n"
284 s += "= What =\n\n"
285 for val in sorted_contributors:
286 s += "<<Anchor(%s)>>\n" % val.name_as_anchor
287 s += val.show_contributions()
288 return s
289
290 def log_revision(self, lr):
291 """Log a revision.
292 :param lr: The LogRevision to be logged.
293 """
294 # We count on always seeing the containing rev before its subrevs.
295 if lr.merge_depth == 0:
296 self.current_top_level_rev = ContainerRevision(lr)
297 else:
298 self.current_top_level_rev.add_subrev(lr)
299 ex_cons = get_ex_cons(lr.rev.get_apparent_authors(), self.all_ex_cons)
300 for ec in ex_cons:
301 ec.add_top_level_revision(self.current_top_level_rev)
302
303
304### TODO: is this really necessary? See bzrlib/log.py.
305log.log_formatter_registry.register('external_contributors', LogExCons,
306 'Find non-Canonical contributors.')
307
308
309def usage():
310 print __doc__
311
312
313page_intro = """This page shows contributions to Launchpad from outside Canonical. It only lists changes that have landed in the Launchpad ''devel'' tree, so changes that land in ''db-devel'' first may take a while to show up (see the [[Trunk|trunk explanation]] for more).
314
315~-''Note for maintainers: this page is updated every 10 minutes by a cron job running as kfogel on devpad (though if there are no new contributions, the page's timestamp won't change). The code that generates this page is [[http://bazaar.launchpad.net/%7Elaunchpad-pqm/launchpad/devel/annotate/head%3A/utilities/community-contributions.py|utilities/community-contributions.py]] in the Launchpad tree.''-~
316
317"""
318
319def main():
320 quiet = False
321 target = None
322 dry_run = False
323
324 if len(sys.argv) < 2:
325 usage()
326 sys.exit(1)
327
328 try:
329 opts, args = getopt.getopt(sys.argv[1:], '?hq',
330 ['help', 'usage', 'dry-run'])
331 except getopt.GetoptError, e:
332 sys.stderr.write("ERROR: " + str(e) + '\n\n')
333 usage()
334 sys.exit(1)
335
336 for opt, value in opts:
337 if opt == '--help' or opt == '-h' or opt == '-?' or opt == 'usage':
338 usage()
339 sys.exit(0)
340 elif opt == '-q' or opt == '--quiet':
341 quiet = True
342 elif opt == '--dry-run':
343 dry_run = True
344
345 # Ensure we have the arguments we need.
346 if len(args) < 1:
347 sys.stderr.write("ERROR: path to Launchpad branch required as argument\n")
348 usage()
349 sys.exit(1)
350
351 target = args[0]
352
353 # Do everything.
354 b = Branch.open(target)
355
356 # ### TODO: 8976 is the first non-Canonical contribution on 'devel'.
357 # On 'db-devel', the magic revision number is 8327. We're aiming at
358 # 'devel' right now, but perhaps it would be good to parameterize
359 # this, or just auto-detect the branch and choose the right number.
360 logger = log.Logger(b, {'start_revision' : 8976,
361 'direction' : 'reverse',
362 'levels' : 0, })
363 lec = LogExCons()
364 if not quiet:
365 print "Calculating (this may take a while)..."
366 logger.show(lec) # This won't "show" anything; it's just for gathering data.
367 page_contents = page_intro + lec.result()
368 def update_if_modified(moinfile):
369 if moinfile._unescape(moinfile.body) == page_contents:
370 return 0 # Nothing changed, so cancel the edit.
371 else:
372 moinfile.body = page_contents
373 return 1
374 if not dry_run:
375 if not quiet:
376 print "Updating wiki..."
377 # Not sure how to get editmoin to obey our quiet flag.
378 editshortcut("https://dev.launchpad.net/Contributions",
379 editfile_func=update_if_modified)
380 if not quiet:
381 print "Done updating wiki."
382 else:
383 print page_contents
384
385
386if __name__ == '__main__':
387 main()