Launchpad itself

Merge lp:~mwhudson/launchpad/no-hosted-area-make-puller-call-branchChanged into lp:launchpad/db-devel

  1. no-hosted-area-make-puller-call-branchChanged
  2. Merge into db-devel
Proposed by Michael Hudson-Doyle
Status: Merged
Approved by: Michael Hudson-Doyle
Approved revision: no longer in the source branch.
Merged at revision: not available
Proposed branch: lp:~mwhudson/launchpad/no-hosted-area-make-puller-call-branchChanged
Merge into: lp:launchpad/db-devel
Prerequisite: lp:~mwhudson/launchpad/no-hosted-area-branchChanged-on-branch
Diff against target: 956 lines (+130/-400)
12 files modified
lib/lp/code/interfaces/codehosting.py (+0/-13)
lib/lp/code/model/branch.py (+5/-2)
lib/lp/code/xmlrpc/codehosting.py (+4/-24)
lib/lp/code/xmlrpc/tests/test_codehosting.py (+11/-60)
lib/lp/codehosting/inmemory.py (+0/-19)
lib/lp/codehosting/puller/scheduler.py (+12/-21)
lib/lp/codehosting/puller/tests/test_scheduler.py (+36/-51)
lib/lp/codehosting/puller/tests/test_worker.py (+26/-26)
lib/lp/codehosting/puller/tests/test_worker_formats.py (+1/-2)
lib/lp/codehosting/puller/worker.py (+35/-13)
lib/lp/codehosting/scanner/bzrsync.py (+0/-46)
lib/lp/codehosting/scanner/tests/test_formats.py (+0/-123)
To merge this branch: bzr merge lp:~mwhudson/launchpad/no-hosted-area-make-puller-call-branchChanged
Reviewer Review Type Date Requested Status
Tim Penhey (community) Approve
Review via email: mp+23810@code.launchpad.net

Description of the change

Hi, this branch changes the puller to call the branchChanged method -- and so record the branch format -- rather than mirrorComplete.

It also removes some cruft -- the setStackedOn endpoint method and the format setting in the scanner -- but not all of it as mirrorComplete is called in many many tests so I'll do that in a later pipe.

To post a comment you must log in.
Revision history for this message
Tim Penhey (thumper) wrote :

Awesome, I can't wait until we land all this.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/code/interfaces/codehosting.py'
2--- lib/lp/code/interfaces/codehosting.py 2010-04-27 02:14:33 +0000
3+++ lib/lp/code/interfaces/codehosting.py 2010-04-27 02:14:37 +0000
4@@ -132,19 +132,6 @@
5 :returns: True if the ScriptActivity record was successfully inserted.
6 """
7
8- def setStackedOn(branch_id, stacked_on_location):
9- """Mark a branch as being stacked on another branch.
10-
11- :param branch_id: The database ID of the stacked branch.
12- :param stacked_on_location: The location of the stacked-on branch.
13- For hosted branches, this is normally '/~foo/bar/baz' where
14- '~foo/bar/baz' is the unique name of another branch.
15- :return: True if the stacked branch information was set successfully.
16- `NoBranchWithID` fault if there's no branch with the given id.
17- `NoSuchBranch` fault if there's no branch matching
18- 'stacked_on_location'.
19- """
20-
21 def createBranch(login_id, branch_path):
22 """Register a new hosted branch in Launchpad.
23
24
25=== modified file 'lib/lp/code/model/branch.py'
26--- lib/lp/code/model/branch.py 2010-04-27 02:14:33 +0000
27+++ lib/lp/code/model/branch.py 2010-04-27 02:14:37 +0000
28@@ -912,14 +912,17 @@
29 self.mirror_status_message = (
30 'Invalid stacked on location: ' + stacked_on_location)
31 self.stacked_on = stacked_on_branch
32- self.last_mirrored = UTC_NOW
33+ if self.branch_type == BranchType.HOSTED:
34+ self.last_mirrored = UTC_NOW
35+ else:
36+ self.last_mirrored = self.last_mirror_attempt
37 self.mirror_failures = 0
38 if (self.next_mirror_time is None
39 and self.branch_type == BranchType.MIRRORED):
40 # No mirror was requested since we started mirroring.
41 increment = getUtility(IBranchPuller).MIRROR_TIME_INCREMENT
42 self.next_mirror_time = (
43- datetime.datetime.now(pytz.timezone('UTC')) + increment)
44+ datetime.now(pytz.timezone('UTC')) + increment)
45 if self.last_mirrored_id != last_revision_id:
46 self.last_mirrored_id = last_revision_id
47 from lp.code.model.branchjob import BranchScanJob
48
49=== modified file 'lib/lp/code/xmlrpc/codehosting.py'
50--- lib/lp/code/xmlrpc/codehosting.py 2010-04-27 02:14:33 +0000
51+++ lib/lp/code/xmlrpc/codehosting.py 2010-04-27 02:14:37 +0000
52@@ -161,29 +161,6 @@
53 removeSecurityProxy(branch).startMirroring()
54 return True
55
56- def setStackedOn(self, branch_id, stacked_on_location):
57- """See `ICodehostingAPI`."""
58- # We don't want the security proxy on the branch set because this
59- # method should be able to see all branches and set stacking
60- # information on any of them.
61- branch_set = removeSecurityProxy(getUtility(IBranchLookup))
62- if stacked_on_location == '':
63- stacked_on_branch = None
64- else:
65- if stacked_on_location.startswith('/'):
66- stacked_on_branch = branch_set.getByUniqueName(
67- stacked_on_location.strip('/'))
68- else:
69- stacked_on_branch = branch_set.getByUrl(
70- stacked_on_location.rstrip('/'))
71- if stacked_on_branch is None:
72- return faults.NoSuchBranch(stacked_on_location)
73- stacked_branch = branch_set.get(branch_id)
74- if stacked_branch is None:
75- return faults.NoBranchWithID(branch_id)
76- stacked_branch.stacked_on = stacked_on_branch
77- return True
78-
79 def createBranch(self, login_id, branch_path):
80 """See `ICodehostingAPI`."""
81 def create_branch(requester):
82@@ -242,7 +219,7 @@
83 last_revision_id, control_string, branch_string,
84 repository_string):
85 """See `ICodehostingAPI`."""
86- def branch_changed(request_mirror):
87+ def branch_changed(requester):
88 branch_set = getUtility(IBranchLookup)
89 branch = branch_set.get(branch_id)
90 if branch is None:
91@@ -263,6 +240,9 @@
92 RepositoryFormat, repository_string,
93 RepositoryFormat.UNRECOGNIZED)
94
95+ if requester == LAUNCHPAD_SERVICES:
96+ branch = removeSecurityProxy(branch)
97+
98 branch.branchChanged(
99 stacked_on_location, last_revision_id, control_format,
100 branch_format, repository_format)
101
102=== modified file 'lib/lp/code/xmlrpc/tests/test_codehosting.py'
103--- lib/lp/code/xmlrpc/tests/test_codehosting.py 2010-04-27 02:14:33 +0000
104+++ lib/lp/code/xmlrpc/tests/test_codehosting.py 2010-04-27 02:14:37 +0000
105@@ -32,7 +32,6 @@
106 from lp.code.enums import BranchType
107 from lp.code.errors import UnknownBranchTypeError
108 from lp.code.interfaces.branch import BRANCH_NAME_VALIDATION_ERROR_MESSAGE
109-from lp.code.interfaces.branchjob import IBranchScanJobSource
110 from lp.code.interfaces.branchlookup import IBranchLookup
111 from lp.code.interfaces.branchtarget import IBranchTarget
112 from lp.code.model.tests.test_branchpuller import AcquireBranchToPullTests
113@@ -286,65 +285,6 @@
114 self.assertEqual(started, activity.date_started)
115 self.assertEqual(completed, activity.date_completed)
116
117- def test_setStackedOnDefaultURLFragment(self):
118- # setStackedOn records that one branch is stacked on another. One way
119- # to find the stacked-on branch is by the URL fragment that's
120- # generated as part of Launchpad's default stacking.
121- stacked_branch = self.factory.makeAnyBranch()
122- stacked_on_branch = self.factory.makeAnyBranch()
123- self.codehosting_api.setStackedOn(
124- stacked_branch.id, '/%s' % stacked_on_branch.unique_name)
125- self.assertEqual(stacked_branch.stacked_on, stacked_on_branch)
126-
127- def test_setStackedOnExternalURL(self):
128- # If setStackedOn is passed an external URL, rather than a URL
129- # fragment, it will mark the branch as being stacked on the branch in
130- # Launchpad registered with that external URL.
131- stacked_branch = self.factory.makeAnyBranch()
132- stacked_on_branch = self.factory.makeAnyBranch(
133- branch_type=BranchType.MIRRORED)
134- self.codehosting_api.setStackedOn(
135- stacked_branch.id, stacked_on_branch.url)
136- self.assertEqual(stacked_branch.stacked_on, stacked_on_branch)
137-
138- def test_setStackedOnExternalURLWithTrailingSlash(self):
139- # If setStackedOn is passed an external URL with a trailing slash, it
140- # won't make a big deal out of it, it will treat it like any other
141- # URL.
142- stacked_branch = self.factory.makeAnyBranch()
143- stacked_on_branch = self.factory.makeAnyBranch(
144- branch_type=BranchType.MIRRORED)
145- url = stacked_on_branch.url + '/'
146- self.codehosting_api.setStackedOn(stacked_branch.id, url)
147- self.assertEqual(stacked_branch.stacked_on, stacked_on_branch)
148-
149- def test_setStackedOnNothing(self):
150- # If setStackedOn is passed an empty string as a stacked-on location,
151- # the branch is marked as not being stacked on any branch.
152- stacked_on_branch = self.factory.makeAnyBranch()
153- stacked_branch = self.factory.makeAnyBranch(
154- stacked_on=stacked_on_branch)
155- self.codehosting_api.setStackedOn(stacked_branch.id, '')
156- self.assertIs(stacked_branch.stacked_on, None)
157-
158- def test_setStackedOnBranchNotFound(self):
159- # If setStackedOn can't find a branch for the given location, it will
160- # return a Fault.
161- stacked_branch = self.factory.makeAnyBranch()
162- url = self.factory.getUniqueURL()
163- fault = self.codehosting_api.setStackedOn(stacked_branch.id, url)
164- self.assertEqual(faults.NoSuchBranch(url), fault)
165-
166- def test_setStackedOnNoBranchWithID(self):
167- # If setStackedOn is called for a branch that doesn't exist, it will
168- # return a Fault.
169- stacked_on_branch = self.factory.makeAnyBranch(
170- branch_type=BranchType.MIRRORED)
171- branch_id = self.getUnusedBranchID()
172- fault = self.codehosting_api.setStackedOn(
173- branch_id, stacked_on_branch.url)
174- self.assertEqual(faults.NoBranchWithID(branch_id), fault)
175-
176 def test_createBranch(self):
177 # createBranch creates a branch with the supplied details and the
178 # caller as registrant.
179@@ -588,6 +528,17 @@
180 login(ANONYMOUS)
181 self.assertEqual(revid, branch.last_mirrored_id)
182
183+ def test_branchChanged_with_LAUNCHPAD_SERVICES(self):
184+ # If you pass LAUNCHPAD_SERVICES as the user id to branchChanged, it
185+ # edits any branch.
186+ revid = self.factory.getUniqueString()
187+ branch = self.factory.makeAnyBranch()
188+ self.codehosting_api.branchChanged(
189+ LAUNCHPAD_SERVICES, branch.id, '', revid,
190+ *self.arbitrary_format_strings)
191+ login(ANONYMOUS)
192+ self.assertEqual(revid, branch.last_mirrored_id)
193+
194 def test_branchChanged_fault_on_unknown_id(self):
195 # If the id passed in doesn't match an existing branch, the fault
196 # "NoBranchWithID" is returned.
197
198=== modified file 'lib/lp/codehosting/inmemory.py'
199--- lib/lp/codehosting/inmemory.py 2010-04-27 02:14:33 +0000
200+++ lib/lp/codehosting/inmemory.py 2010-04-27 02:14:37 +0000
201@@ -527,25 +527,6 @@
202 FakeScriptActivity(name, hostname, date_started, date_completed))
203 return True
204
205- def setStackedOn(self, branch_id, stacked_on_location):
206- branch = self._branch_set.get(branch_id)
207- if branch is None:
208- return faults.NoBranchWithID(branch_id)
209- if stacked_on_location == '':
210- branch.stacked_on = None
211- return True
212- stacked_on_location = stacked_on_location.rstrip('/')
213- for stacked_on_branch in self._branch_set:
214- if stacked_on_location == stacked_on_branch.url:
215- branch.stacked_on = stacked_on_branch
216- break
217- if stacked_on_location == '/' + stacked_on_branch.unique_name:
218- branch.stacked_on = stacked_on_branch
219- break
220- else:
221- return faults.NoSuchBranch(stacked_on_location)
222- return True
223-
224 def createBranch(self, requester_id, branch_path):
225 if not branch_path.startswith('/'):
226 return faults.InvalidPath(branch_path)
227
228=== modified file 'lib/lp/codehosting/puller/scheduler.py'
229--- lib/lp/codehosting/puller/scheduler.py 2010-04-27 02:14:33 +0000
230+++ lib/lp/codehosting/puller/scheduler.py 2010-04-27 02:14:37 +0000
231@@ -22,14 +22,13 @@
232
233 from contrib.glock import GlobalLock, LockAlreadyAcquired
234
235-import canonical
236 from canonical.cachedproperty import cachedproperty
237+from lp.code.interfaces.codehosting import LAUNCHPAD_SERVICES
238 from lp.codehosting.puller.worker import (
239 get_canonical_url_for_branch_name)
240 from lp.codehosting.puller import get_lock_id_for_branch_id
241 from canonical.config import config
242 from canonical.launchpad.webapp import errorlog
243-from canonical.launchpad.xmlrpc import faults
244 from lp.services.twistedsupport.processmonitor import (
245 ProcessMonitorProtocolWithTimeout)
246 from lp.services.twistedsupport.task import (
247@@ -226,20 +225,19 @@
248 def errReceived(self, data):
249 self._stderr.write(data)
250
251- def do_setStackedOn(self, stacked_on_location):
252- self.runNotification(self.listener.setStackedOn, stacked_on_location)
253-
254 def do_startMirroring(self):
255 self.resetTimeout()
256 self.runNotification(self.listener.startMirroring)
257
258- def do_mirrorSucceeded(self, revid_before, revid_after):
259- def mirrorSucceeded():
260+ def do_branchChanged(self, stacked_on_url, revid_before, revid_after,
261+ control_string, branch_string, repository_string):
262+ def branchChanged():
263 d = defer.maybeDeferred(
264- self.listener.mirrorSucceeded, revid_before, revid_after)
265+ self.listener.branchChanged, stacked_on_url, revid_before,
266+ revid_after, control_string, branch_string, repository_string)
267 d.addCallback(self.reportMirrorFinished)
268 return d
269- self.runNotification(mirrorSucceeded)
270+ self.runNotification(branchChanged)
271
272 def do_mirrorFailed(self, reason, oops):
273 def mirrorFailed():
274@@ -347,16 +345,6 @@
275 deferred.addBoth(self.releaseOopsPrefix)
276 return deferred
277
278- def setStackedOn(self, stacked_on_location):
279- deferred = self.codehosting_endpoint.callRemote(
280- 'setStackedOn', self.branch_id, stacked_on_location)
281- def no_such_branch(failure):
282- # If there's no branch for stacked_on_location, then we just
283- # swallow the error. It's ok for branches to be stacked on
284- # branches that Launchpad doesn't know about.
285- failure.trap(faults.NoSuchBranch)
286- return deferred.addErrback(no_such_branch)
287-
288 def startMirroring(self):
289 self.logger.info(
290 'Worker started on branch %d: %s to %s', self.branch_id,
291@@ -368,7 +356,8 @@
292 return self.codehosting_endpoint.callRemote(
293 'mirrorFailed', self.branch_id, reason)
294
295- def mirrorSucceeded(self, revid_before, revid_after):
296+ def branchChanged(self, stacked_on_url, revid_before, revid_after,
297+ control_string, branch_string, repository_string):
298 if revid_before == revid_after:
299 was_noop = 'noop'
300 else:
301@@ -378,7 +367,9 @@
302 ' (%s)', self.branch_type_name, self.branch_id, self.source_url,
303 self.destination_url, revid_before, revid_after, was_noop)
304 return self.codehosting_endpoint.callRemote(
305- 'mirrorComplete', self.branch_id, revid_after)
306+ 'branchChanged', LAUNCHPAD_SERVICES, self.branch_id,
307+ stacked_on_url, revid_after, control_string, branch_string,
308+ repository_string)
309
310 def log(self, message):
311 self.logger.info('From worker: %s', message)
312
313=== modified file 'lib/lp/codehosting/puller/tests/test_scheduler.py'
314--- lib/lp/codehosting/puller/tests/test_scheduler.py 2010-04-27 02:14:33 +0000
315+++ lib/lp/codehosting/puller/tests/test_scheduler.py 2010-04-27 02:14:37 +0000
316@@ -14,7 +14,7 @@
317 import pytz
318
319 from bzrlib.branch import Branch
320-from bzrlib.bzrdir import BzrDir
321+from bzrlib.bzrdir import BzrDir, format_registry
322 from bzrlib.urlutils import join as urljoin
323
324 from twisted.internet import defer, error, reactor
325@@ -26,7 +26,6 @@
326
327 from canonical.config import config
328 from canonical.launchpad.webapp import errorlog
329-from canonical.launchpad.xmlrpc import faults
330 from canonical.testing import (
331 reset_logging, TwistedLayer, TwistedAppServerLayer)
332 from lp.codehosting.puller import get_lock_id_for_branch_id, scheduler
333@@ -60,14 +59,6 @@
334 def _remote_acquireBranchToPull(self, *args):
335 return defer.succeed(0)
336
337- def _remote_setStackedOn(self, branch_id, stacked_on_location):
338- if stacked_on_location == 'raise-branch-not-found':
339- try:
340- raise faults.NoSuchBranch(stacked_on_location)
341- except faults.NoSuchBranch:
342- return defer.fail()
343- return defer.succeed(None)
344-
345
346 class TestJobScheduler(TrialTestCase):
347
348@@ -226,14 +217,14 @@
349 def __init__(self):
350 self.calls = []
351
352- def setStackedOn(self, stacked_on_location):
353- self.calls.append(('setStackedOn', stacked_on_location))
354-
355 def startMirroring(self):
356 self.calls.append('startMirroring')
357
358- def mirrorSucceeded(self, revid_before, revid_after):
359- self.calls.append(('mirrorSucceeded', revid_before, revid_after))
360+ def branchChanged(self, stacked_on_url, revid_before, revid_after,
361+ control_string, branch_string, repository_string):
362+ self.calls.append(
363+ ('branchChanged', stacked_on_url, revid_before, revid_after,
364+ control_string, branch_string, repository_string))
365
366 def mirrorFailed(self, message, oops):
367 self.calls.append(('mirrorFailed', message, oops))
368@@ -259,20 +250,14 @@
369 self.assertEqual(['startMirroring'], self.listener.calls)
370 self.assertProtocolSuccess()
371
372- def test_setStackedOn(self):
373- # Receiving a setStackedOn message notifies the listener.
374- self.protocol.do_setStackedOn('/~foo/bar/baz')
375- self.assertEqual(
376- [('setStackedOn', '/~foo/bar/baz')], self.listener.calls)
377- self.assertProtocolSuccess()
378-
379- def test_mirrorSucceeded(self):
380- """Receiving a mirrorSucceeded message notifies the listener."""
381+ def test_branchChanged(self):
382+ """Receiving a branchChanged message notifies the listener."""
383 self.protocol.do_startMirroring()
384 self.listener.calls = []
385- self.protocol.do_mirrorSucceeded('rev1', 'rev2')
386+ self.protocol.do_branchChanged('', 'rev1', 'rev2', '', '', '')
387 self.assertEqual(
388- [('mirrorSucceeded', 'rev1', 'rev2')], self.listener.calls)
389+ [('branchChanged', '', 'rev1', 'rev2', '', '', '')],
390+ self.listener.calls)
391 self.assertProtocolSuccess()
392
393 def test_mirrorFailed(self):
394@@ -309,8 +294,8 @@
395 """Receiving 'startMirroring' resets the timeout."""
396 self.assertMessageResetsTimeout(self.protocol.do_startMirroring)
397
398- def test_mirrorSucceededDoesNotResetTimeout(self):
399- """Receiving 'mirrorSucceeded' doesn't reset the timeout.
400+ def test_branchChangedDoesNotResetTimeout(self):
401+ """Receiving 'branchChanged' doesn't reset the timeout.
402
403 It's possible that in pathological cases, the worker process might
404 hang around even after it has said that it's finished. When that
405@@ -319,7 +304,7 @@
406 """
407 self.protocol.do_startMirroring()
408 self.clock.advance(config.supermirror.worker_timeout - 1)
409- self.protocol.do_mirrorSucceeded('rev1', 'rev2')
410+ self.protocol.do_branchChanged('', '', '', '', '', '')
411 self.clock.advance(2)
412 return self.assertFailure(
413 self.termination_deferred, error.TimeoutError)
414@@ -428,6 +413,7 @@
415 self.arbitrary_branch_id, 'arbitrary-source', 'arbitrary-dest',
416 BranchType.HOSTED, None, logging.getLogger(), self.status_client,
417 set(['oops-prefix']))
418+ self.factory = ObjectFactory()
419
420 def test_unexpectedError(self):
421 """The puller master logs an OOPS when it receives an unexpected
422@@ -453,31 +439,24 @@
423
424 return deferred.addCallback(checkMirrorStarted)
425
426- def test_setStackedOn(self):
427- stacked_on_location = '/~foo/bar/baz'
428- deferred = self.eventHandler.setStackedOn(stacked_on_location)
429-
430- def checkSetStackedOn(ignored):
431- self.assertEqual(
432- [('setStackedOn', self.arbitrary_branch_id,
433- stacked_on_location)],
434- self.status_client.calls)
435-
436- return deferred.addCallback(checkSetStackedOn)
437-
438- def test_mirrorComplete(self):
439- arbitrary_revision_ids = ('rev1', 'rev2')
440+ def test_branchChanged(self):
441+ (stacked_on_url, revid_before, revid_after, control_string,
442+ branch_string, repository_string
443+ ) = list(self.factory.getUniqueString() for i in range(6))
444 deferred = defer.maybeDeferred(self.eventHandler.startMirroring)
445
446- def mirrorSucceeded(*ignored):
447+ def branchChanged(*ignored):
448 self.status_client.calls = []
449- return self.eventHandler.mirrorSucceeded(*arbitrary_revision_ids)
450- deferred.addCallback(mirrorSucceeded)
451+ return self.eventHandler.branchChanged(
452+ stacked_on_url, revid_before, revid_after, control_string,
453+ branch_string, repository_string)
454+ deferred.addCallback(branchChanged)
455
456 def checkMirrorCompleted(ignored):
457 self.assertEqual(
458- [('mirrorComplete', self.arbitrary_branch_id,
459- arbitrary_revision_ids[1])],
460+ [('branchChanged', self.arbitrary_branch_id, stacked_on_url,
461+ revid_after, control_string, branch_string,
462+ repository_string)],
463 self.status_client.calls)
464 return deferred.addCallback(checkMirrorCompleted)
465
466@@ -694,9 +673,15 @@
467 deferred = puller_master.mirror()
468
469 def check_authserver_called(ignored):
470+ default_format = format_registry.get('default')()
471+ control_string = default_format.get_format_string()
472+ branch_string = \
473+ default_format.get_branch_format().get_format_string()
474+ repository_string = \
475+ default_format.repository_format.get_format_string()
476 self.assertEqual(
477- [('setStackedOn', self.db_branch.id, ''),
478- ('mirrorComplete', self.db_branch.id, revision_id)],
479+ [('branchChanged', self.db_branch.id, '', revision_id,
480+ control_string, branch_string, repository_string)],
481 self.client.calls)
482 return ignored
483 deferred.addCallback(check_authserver_called)
484@@ -775,7 +760,7 @@
485
486 check_lock_id_script = """
487 branch.lock_write()
488- protocol.mirrorSucceeded('a', 'b')
489+ protocol.mirrorFailed('a', 'b')
490 protocol.sendEvent(
491 'lock_id', branch.control_files._lock.peek()['user'])
492 sys.stdout.flush()
493
494=== modified file 'lib/lp/codehosting/puller/tests/test_worker.py'
495--- lib/lp/codehosting/puller/tests/test_worker.py 2010-04-27 02:14:33 +0000
496+++ lib/lp/codehosting/puller/tests/test_worker.py 2010-04-27 02:14:37 +0000
497@@ -31,7 +31,7 @@
498 ImportedBranchPolicy, MirroredBranchPolicy)
499 from lp.code.enums import BranchType
500 from lp.testing import TestCase
501-from lp.testing.factory import LaunchpadObjectFactory
502+from lp.testing.factory import LaunchpadObjectFactory, ObjectFactory
503
504
505 def get_netstrings(line):
506@@ -202,6 +202,11 @@
507 source_branch.repository._format,
508 mirrored_branch.repository._format)
509
510+ def getStackedOnUrlFromNetStringOutput(self, netstring_output):
511+ netstrings = get_netstrings(netstring_output)
512+ branchChanged_index = netstrings.index('branchChanged')
513+ return netstrings[branchChanged_index + 2]
514+
515 def testSendsStackedInfo(self):
516 # When the puller worker stacks a branch, it reports the stacked on
517 # URL to the master.
518@@ -212,37 +217,36 @@
519 stacked_branch.base, self.get_url('destdir'),
520 protocol=PullerWorkerProtocol(protocol_output),
521 policy=PrearrangedStackedBranchPolicy(base_branch.base))
522- to_mirror.mirrorWithoutChecks()
523- self.assertEqual(
524- ['setStackedOn', str(to_mirror.branch_id), base_branch.base],
525- get_netstrings(protocol_output.getvalue()))
526+ to_mirror.mirror()
527+ stacked_on_url = self.getStackedOnUrlFromNetStringOutput(
528+ protocol_output.getvalue())
529+ self.assertEqual(base_branch.base, stacked_on_url)
530
531 def testDoesntSendStackedInfoUnstackableFormat(self):
532 # Mirroring an unstackable branch sends '' as the stacked-on location
533 # to the master.
534- source_branch = self.make_branch('source-branch')
535+ source_branch = self.make_branch('source-branch', format='pack-0.92')
536 protocol_output = StringIO()
537 to_mirror = self.makePullerWorker(
538 source_branch.base, self.get_url('destdir'),
539 protocol=PullerWorkerProtocol(protocol_output))
540- to_mirror.mirrorWithoutChecks()
541- self.assertEqual(
542- ['setStackedOn', str(to_mirror.branch_id), ''],
543- get_netstrings(protocol_output.getvalue()))
544+ to_mirror.mirror()
545+ stacked_on_url = self.getStackedOnUrlFromNetStringOutput(
546+ protocol_output.getvalue())
547+ self.assertEqual('', stacked_on_url)
548
549 def testDoesntSendStackedInfoNotStacked(self):
550 # Mirroring a non-stacked branch sends '' as the stacked-on location
551 # to the master.
552- source_branch = self.make_branch(
553- 'source-branch', format='1.9')
554+ source_branch = self.make_branch('source-branch', format='1.9')
555 protocol_output = StringIO()
556 to_mirror = self.makePullerWorker(
557 source_branch.base, self.get_url('destdir'),
558 protocol=PullerWorkerProtocol(protocol_output))
559- to_mirror.mirrorWithoutChecks()
560- self.assertEqual(
561- ['setStackedOn', str(to_mirror.branch_id), ''],
562- get_netstrings(protocol_output.getvalue()))
563+ to_mirror.mirror()
564+ stacked_on_url = self.getStackedOnUrlFromNetStringOutput(
565+ protocol_output.getvalue())
566+ self.assertEqual('', stacked_on_url)
567
568
569 class TestBranchMirrorerCheckAndFollowBranchReference(TestCase):
570@@ -591,6 +595,7 @@
571 TestCaseInTempDir.setUp(self)
572 self.output = StringIO()
573 self.protocol = PullerWorkerProtocol(self.output)
574+ self.factory = ObjectFactory()
575
576 def assertSentNetstrings(self, expected_netstrings):
577 """Assert that the protocol sent the given netstrings (in order)."""
578@@ -612,12 +617,13 @@
579 self.protocol.startMirroring()
580 self.assertSentNetstrings(['startMirroring', '0'])
581
582- def test_mirrorSucceeded(self):
583- # Calling 'mirrorSucceeded' sends the revids and 'mirrorSucceeded'.
584+ def test_branchChanged(self):
585+ # Calling 'branchChanged' sends the arguments.
586+ arbitrary_args = [self.factory.getUniqueString() for x in range(6)]
587 self.protocol.startMirroring()
588 self.resetBuffers()
589- self.protocol.mirrorSucceeded('rev1', 'rev2')
590- self.assertSentNetstrings(['mirrorSucceeded', '2', 'rev1', 'rev2'])
591+ self.protocol.branchChanged(*arbitrary_args)
592+ self.assertSentNetstrings(['branchChanged', '6'] + arbitrary_args)
593
594 def test_mirrorFailed(self):
595 # Calling 'mirrorFailed' sends the error message.
596@@ -633,12 +639,6 @@
597 self.protocol.progressMade('test')
598 self.assertSentNetstrings(['progressMade', '0'])
599
600- def test_setStackedOn(self):
601- # Calling 'setStackedOn' sends the location of the stacked-on branch,
602- # if any.
603- self.protocol.setStackedOn('/~foo/bar/baz')
604- self.assertSentNetstrings(['setStackedOn', '1', '/~foo/bar/baz'])
605-
606 def test_log(self):
607 # Calling 'log' sends 'log' as a netstring and its arguments, after
608 # formatting as a string.
609
610=== modified file 'lib/lp/codehosting/puller/tests/test_worker_formats.py'
611--- lib/lp/codehosting/puller/tests/test_worker_formats.py 2010-04-27 02:14:33 +0000
612+++ lib/lp/codehosting/puller/tests/test_worker_formats.py 2010-04-27 02:14:37 +0000
613@@ -7,7 +7,7 @@
614
615 import unittest
616
617-from bzrlib.branch import Branch, BzrBranchFormat7
618+from bzrlib.branch import Branch
619 from bzrlib.bzrdir import BzrDirFormat6, BzrDirMetaFormat1
620 from bzrlib.repofmt.knitrepo import RepositoryFormatKnit1
621 from bzrlib.repofmt.pack_repo import RepositoryFormatKnitPack5
622@@ -16,7 +16,6 @@
623
624 from lp.codehosting.puller.tests import PullerWorkerMixin
625 from lp.codehosting.tests.helpers import LoomTestMixin
626-from lazr.uri import URI
627
628
629 class TestPullerWorkerFormats(TestCaseWithRepository, PullerWorkerMixin,
630
631=== modified file 'lib/lp/codehosting/puller/worker.py'
632--- lib/lp/codehosting/puller/worker.py 2010-04-27 02:14:33 +0000
633+++ lib/lp/codehosting/puller/worker.py 2010-04-27 02:14:37 +0000
634@@ -8,9 +8,11 @@
635 import sys
636 import urllib2
637
638-from bzrlib.branch import Branch
639+from bzrlib.branch import Branch, BzrBranchFormat4
640 from bzrlib.bzrdir import BzrDir
641 from bzrlib import errors
642+from bzrlib.repofmt.weaverepo import (
643+ RepositoryFormat4, RepositoryFormat5, RepositoryFormat6)
644 from bzrlib.ui import SilentUIFactory
645 import bzrlib.ui
646
647@@ -21,6 +23,7 @@
648 from lp.codehosting.puller import get_lock_id_for_branch_id
649 from lp.codehosting.vfs.branchfs import (
650 BadUrlLaunchpad, BadUrlScheme, BadUrlSsh, make_branch_mirrorer)
651+from lp.code.bzr import BranchFormat, RepositoryFormat
652 from lp.code.enums import BranchType
653 from lazr.uri import InvalidURIError
654
655@@ -69,7 +72,7 @@
656 """The protocol used to communicate with the puller scheduler.
657
658 This protocol notifies the scheduler of events such as startMirroring,
659- mirrorSucceeded and mirrorFailed.
660+ branchChanged and mirrorFailed.
661 """
662
663 def __init__(self, output):
664@@ -84,14 +87,14 @@
665 for argument in args:
666 self.sendNetstring(str(argument))
667
668- def setStackedOn(self, stacked_on_location):
669- self.sendEvent('setStackedOn', stacked_on_location)
670-
671 def startMirroring(self):
672 self.sendEvent('startMirroring')
673
674- def mirrorSucceeded(self, revid_before, revid_after):
675- self.sendEvent('mirrorSucceeded', revid_before, revid_after)
676+ def branchChanged(self, stacked_on_url, revid_before, revid_after,
677+ control_string, branch_string, repository_string):
678+ self.sendEvent(
679+ 'branchChanged', stacked_on_url, revid_before, revid_after,
680+ control_string, branch_string, repository_string)
681
682 def mirrorFailed(self, message, oops_id):
683 self.sendEvent('mirrorFailed', message, oops_id)
684@@ -264,9 +267,8 @@
685 # We use stacked_on_url == '' to mean "no stacked on location"
686 # because XML-RPC doesn't support None.
687 stacked_on_url = ''
688- if self.protocol is not None:
689- self.protocol.setStackedOn(stacked_on_url)
690 dest_branch.pull(source_branch, overwrite=True)
691+ return stacked_on_url
692
693 def mirror(self, source_branch, destination_url):
694 """Mirror 'source_branch' to 'destination_url'."""
695@@ -279,8 +281,8 @@
696 # (currently 5 minutes).
697 if branch.get_physical_lock_status():
698 branch.break_lock()
699- self.updateBranch(source_branch, branch)
700- return branch, revid_before
701+ stacked_on_url = self.updateBranch(source_branch, branch)
702+ return branch, revid_before, stacked_on_url
703
704
705 class PullerWorker:
706@@ -388,7 +390,8 @@
707 """
708 self.protocol.startMirroring()
709 try:
710- dest_branch, revid_before = self.mirrorWithoutChecks()
711+ dest_branch, revid_before, stacked_on_url = \
712+ self.mirrorWithoutChecks()
713 # add further encountered errors from the production runs here
714 # ------ HERE ---------
715 #
716@@ -457,7 +460,26 @@
717
718 else:
719 revid_after = dest_branch.last_revision()
720- self.protocol.mirrorSucceeded(revid_before, revid_after)
721+ # XXX: Aaron Bentley 2008-06-13
722+ # Bazaar does not provide a public API for learning about
723+ # format markers. Fix this in Bazaar, then here.
724+ control_string = dest_branch.bzrdir._format.get_format_string()
725+ if dest_branch._format.__class__ is BzrBranchFormat4:
726+ branch_string = BranchFormat.BZR_BRANCH_4.title
727+ else:
728+ branch_string = dest_branch._format.get_format_string()
729+ repository_format = dest_branch.repository._format
730+ if repository_format.__class__ is RepositoryFormat6:
731+ repository_string = RepositoryFormat.BZR_REPOSITORY_6.title
732+ elif repository_format.__class__ is RepositoryFormat5:
733+ repository_string = RepositoryFormat.BZR_REPOSITORY_5.title
734+ elif repository_format.__class__ is RepositoryFormat4:
735+ repository_string = RepositoryFormat.BZR_REPOSITORY_4.title
736+ else:
737+ repository_string = repository_format.get_format_string()
738+ self.protocol.branchChanged(
739+ stacked_on_url, revid_before, revid_after, control_string,
740+ branch_string, repository_string)
741
742 def __eq__(self, other):
743 return self.source == other.source and self.dest == other.dest
744
745=== modified file 'lib/lp/codehosting/scanner/bzrsync.py'
746--- lib/lp/codehosting/scanner/bzrsync.py 2010-03-02 18:56:25 +0000
747+++ lib/lp/codehosting/scanner/bzrsync.py 2010-04-27 02:14:37 +0000
748@@ -21,9 +21,6 @@
749 from zope.component import getUtility
750 from zope.event import notify
751
752-from bzrlib.branch import BzrBranchFormat4
753-from bzrlib.repofmt.weaverepo import (
754- RepositoryFormat4, RepositoryFormat5, RepositoryFormat6)
755 from bzrlib import urlutils
756
757 from lazr.uri import URI
758@@ -32,7 +29,6 @@
759 from lp.codehosting.puller.worker import BranchMirrorer
760 from lp.codehosting.scanner import events
761 from lp.codehosting.vfs.branchfs import BranchPolicy
762-from lp.code.bzr import BranchFormat, ControlFormat, RepositoryFormat
763 from lp.code.interfaces.branchjob import IRosettaUploadJobSource
764 from lp.code.interfaces.branchrevision import IBranchRevisionSet
765 from lp.code.interfaces.revision import IRevisionSet
766@@ -118,7 +114,6 @@
767 # written to by the branch-scanner, so they are not subject to
768 # write-lock contention. Update them all in a single transaction to
769 # improve the performance and allow garbage collection in the future.
770- self.setFormats(bzr_branch)
771 db_ancestry, db_history, db_branch_revision_map = (
772 self.retrieveDatabaseAncestry())
773
774@@ -183,46 +178,6 @@
775 bzr_history = bzr_branch.revision_history()
776 return bzr_ancestry, bzr_history
777
778- def setFormats(self, bzr_branch):
779- """Record the stored formats in the database object.
780-
781- The previous value is unconditionally overwritten.
782-
783- Note that the strings associated with the formats themselves are used,
784- not the strings on disk.
785- """
786- def match_title(enum, title, default):
787- for value in enum.items:
788- if value.title == title:
789- return value
790- else:
791- return default
792-
793- # XXX: Aaron Bentley 2008-06-13
794- # Bazaar does not provide a public API for learning about format
795- # markers. Fix this in Bazaar, then here.
796- control_string = bzr_branch.bzrdir._format.get_format_string()
797- if bzr_branch._format.__class__ is BzrBranchFormat4:
798- branch_string = BranchFormat.BZR_BRANCH_4.title
799- else:
800- branch_string = bzr_branch._format.get_format_string()
801- repository_format = bzr_branch.repository._format
802- if repository_format.__class__ is RepositoryFormat6:
803- repository_string = RepositoryFormat.BZR_REPOSITORY_6.title
804- elif repository_format.__class__ is RepositoryFormat5:
805- repository_string = RepositoryFormat.BZR_REPOSITORY_5.title
806- elif repository_format.__class__ is RepositoryFormat4:
807- repository_string = RepositoryFormat.BZR_REPOSITORY_4.title
808- else:
809- repository_string = repository_format.get_format_string()
810- self.db_branch.control_format = match_title(
811- ControlFormat, control_string, ControlFormat.UNRECOGNIZED)
812- self.db_branch.branch_format = match_title(
813- BranchFormat, branch_string, BranchFormat.UNRECOGNIZED)
814- self.db_branch.repository_format = match_title(
815- RepositoryFormat, repository_string,
816- RepositoryFormat.UNRECOGNIZED)
817-
818 def planDatabaseChanges(self, bzr_branch, bzr_ancestry, bzr_history,
819 db_ancestry, db_history, db_branch_revision_map):
820 """Plan database changes to synchronize with bzrlib data.
821@@ -333,7 +288,6 @@
822 """Insert a batch of BranchRevision rows."""
823 self.logger.info("Inserting %d branchrevision records.",
824 len(revids_to_insert))
825- revision_set = getUtility(IRevisionSet)
826 revid_seq_pairs = revids_to_insert.items()
827 for revid_seq_pair_chunk in iter_list_chunks(revid_seq_pairs, 1000):
828 self.db_branch.createBranchRevisionFromIDs(revid_seq_pair_chunk)
829
830=== removed file 'lib/lp/codehosting/scanner/tests/test_formats.py'
831--- lib/lp/codehosting/scanner/tests/test_formats.py 2009-09-17 05:11:31 +0000
832+++ lib/lp/codehosting/scanner/tests/test_formats.py 1970-01-01 00:00:00 +0000
833@@ -1,123 +0,0 @@
834-# Copyright 2009 Canonical Ltd. This software is licensed under the
835-# GNU Affero General Public License version 3 (see the file LICENSE).
836-
837-"""Tests for how the scanner processes Bazaar formats."""
838-
839-__metaclass__ = type
840-
841-import unittest
842-
843-from lp.codehosting.scanner.tests.test_bzrsync import BzrSyncTestCase
844-from lp.code.interfaces.branch import (
845- BranchFormat, ControlFormat, RepositoryFormat)
846-
847-
848-class TestScanFormat2a(BzrSyncTestCase):
849- """Test scanning of 2a repositories."""
850-
851- def testRecognize2a(self):
852- """Ensure scanner records correct formats for pack branches."""
853- self.makeBzrSync(self.db_branch).syncBranchAndClose()
854- self.assertEqual(self.db_branch.branch_format,
855- BranchFormat.BZR_BRANCH_7)
856- self.assertEqual(self.db_branch.repository_format,
857- RepositoryFormat.BZR_CHK_2A)
858- self.assertEqual(self.db_branch.control_format,
859- ControlFormat.BZR_METADIR_1)
860-
861-
862-class TestScanFormatPack(BzrSyncTestCase):
863- """Test scanning of pack-format repositories."""
864-
865- def makeBzrBranchAndTree(self, db_branch):
866- return BzrSyncTestCase.makeBzrBranchAndTree(
867- self, db_branch, 'pack-0.92')
868-
869- def testRecognizePack(self):
870- """Ensure scanner records correct formats for pack branches."""
871- self.makeBzrSync(self.db_branch).syncBranchAndClose()
872- self.assertEqual(self.db_branch.branch_format,
873- BranchFormat.BZR_BRANCH_6)
874- self.assertEqual(self.db_branch.repository_format,
875- RepositoryFormat.BZR_KNITPACK_1)
876- self.assertEqual(self.db_branch.control_format,
877- ControlFormat.BZR_METADIR_1)
878-
879-
880-class TestScanFormatKnit(BzrSyncTestCase):
881- """Test scanning of knit-format repositories."""
882-
883- def makeBzrBranchAndTree(self, db_branch):
884- return BzrSyncTestCase.makeBzrBranchAndTree(self, db_branch, 'knit')
885-
886- def testRecognizeKnit(self):
887- """Ensure scanner records correct formats for knit branches."""
888- self.makeBzrSync(self.db_branch).syncBranchAndClose()
889- self.assertEqual(self.db_branch.branch_format,
890- BranchFormat.BZR_BRANCH_5)
891-
892-
893-class TestScanBranchFormat7(BzrSyncTestCase):
894- """Test scanning of format 7 (i.e. stacking-supporting) branches."""
895-
896- def makeBzrBranchAndTree(self, db_branch):
897- return BzrSyncTestCase.makeBzrBranchAndTree(
898- self, db_branch, '1.6')
899-
900- def testRecognizeDevelopment(self):
901- """Ensure scanner records correct format for development branches."""
902- self.makeBzrSync(self.db_branch).syncBranchAndClose()
903- self.assertEqual(
904- self.db_branch.branch_format, BranchFormat.BZR_BRANCH_7)
905-
906-
907-class TestScanFormatWeave(BzrSyncTestCase):
908- """Test scanning of weave-format branches.
909-
910- Weave is an "all-in-one" format, where branch, repo and tree formats are
911- implied by the control directory format."""
912-
913- def makeBzrBranchAndTree(self, db_branch):
914- return BzrSyncTestCase.makeBzrBranchAndTree(self, db_branch, 'weave')
915-
916- def testRecognizeWeave(self):
917- """Ensure scanner records correct weave formats."""
918- self.makeBzrSync(self.db_branch).syncBranchAndClose()
919- self.assertEqual(self.db_branch.branch_format,
920- BranchFormat.BZR_BRANCH_4)
921- self.assertEqual(self.db_branch.repository_format,
922- RepositoryFormat.BZR_REPOSITORY_6)
923- self.assertEqual(self.db_branch.control_format,
924- ControlFormat.BZR_DIR_6)
925-
926-
927-class TestScanUnrecognizedFormat(BzrSyncTestCase):
928- """Test scanning unrecognized formats"""
929-
930- def testUnrecognize(self):
931- """Scanner should record UNRECOGNIZED for all format values."""
932- class MockFormat:
933- def get_format_string(self):
934- return 'Unrecognizable'
935-
936- class MockWithFormat:
937- def __init__(self):
938- self._format = MockFormat()
939-
940- class MockBranch(MockWithFormat):
941- bzrdir = MockWithFormat()
942- repository = MockWithFormat()
943-
944- branch = MockBranch()
945- self.makeBzrSync(self.db_branch).setFormats(branch)
946- self.assertEqual(self.db_branch.branch_format,
947- BranchFormat.UNRECOGNIZED)
948- self.assertEqual(self.db_branch.repository_format,
949- RepositoryFormat.UNRECOGNIZED)
950- self.assertEqual(self.db_branch.control_format,
951- ControlFormat.UNRECOGNIZED)
952-
953-
954-def test_suite():
955- return unittest.TestLoader().loadTestsFromName(__name__)
956-

Subscribers

People subscribed via source and target branches

to status/vote changes: