Launchpad itself

Merge lp:~mwhudson/launchpad/no-hosted-area-puller into lp:launchpad

  1. no-hosted-area-puller
  2. Merge into devel
Proposed by Michael Hudson-Doyle
Status: Merged
Approved by: Michael Hudson-Doyle
Approved revision: no longer in the source branch.
Merged at revision: 10828
Proposed branch: lp:~mwhudson/launchpad/no-hosted-area-puller
Merge into: lp:launchpad
Prerequisite: lp:~mwhudson/launchpad/no-hosted-area-formats-from-codehosting
Diff against target: 1522 lines (+172/-712)
20 files modified
lib/lp/code/configure.zcml (+0/-1)
lib/lp/code/interfaces/branch.py (+0/-7)
lib/lp/code/model/branch.py (+0/-12)
lib/lp/code/model/branchpuller.py (+1/-2)
lib/lp/code/model/directbranchcommit.py (+0/-2)
lib/lp/code/model/tests/test_branch.py (+0/-73)
lib/lp/code/model/tests/test_branchpuller.py (+29/-35)
lib/lp/code/xmlrpc/codehosting.py (+0/-3)
lib/lp/code/xmlrpc/tests/test_codehosting.py (+2/-60)
lib/lp/codehosting/puller/scheduler.py (+4/-10)
lib/lp/codehosting/puller/tests/__init__.py (+16/-1)
lib/lp/codehosting/puller/tests/test_acceptance.py (+71/-185)
lib/lp/codehosting/puller/tests/test_errors.py (+1/-9)
lib/lp/codehosting/puller/tests/test_scheduler.py (+9/-29)
lib/lp/codehosting/puller/tests/test_worker.py (+15/-87)
lib/lp/codehosting/puller/tests/test_worker_formats.py (+0/-47)
lib/lp/codehosting/puller/worker.py (+3/-18)
lib/lp/codehosting/vfs/__init__.py (+2/-2)
lib/lp/codehosting/vfs/branchfs.py (+19/-128)
scripts/mirror-branch.py (+0/-1)
To merge this branch: bzr merge lp:~mwhudson/launchpad/no-hosted-area-puller
Reviewer Review Type Date Requested Status
Tim Penhey (community) Approve
Review via email: mp+23729@code.launchpad.net

Description of the change

Hi again Tim,

This branch changes the puller to adapt to the changes in codehosting -- mostly removing stuff as it will never have to touch a hosted branch again.

It's particularly satisfying to delete all the craziness to do with deferring mirrors of branches that are stacked on branches that do not exist in the hosted area yet. I had no idea how much of this there was!

The branch is large but I think not too complicated.

Cheers,
mwh

To post a comment you must log in.
Revision history for this message
Tim Penhey (thumper) wrote :

Apart from the XXX comments we talked about, this looks good.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== removed directory 'lib/canonical/launchpad/apidoc'
2=== modified file 'lib/lp/code/configure.zcml'
3--- lib/lp/code/configure.zcml 2010-04-27 02:06:51 +0000
4+++ lib/lp/code/configure.zcml 2010-04-27 02:07:14 +0000
5@@ -513,7 +513,6 @@
6 getMainlineBranchRevisions
7 getMergeProposals
8 getStackedBranches
9- getStackedBranchesWithIncompleteMirrors
10 createBranchRevision
11 getTipRevision
12 updateScannedDetails
13
14=== modified file 'lib/lp/code/interfaces/branch.py'
15--- lib/lp/code/interfaces/branch.py 2010-04-13 11:05:58 +0000
16+++ lib/lp/code/interfaces/branch.py 2010-04-27 02:07:14 +0000
17@@ -818,13 +818,6 @@
18 def getStackedBranches():
19 """The branches that are stacked on this one."""
20
21- def getStackedBranchesWithIncompleteMirrors():
22- """Branches that are stacked on this one but aren't done mirroring.
23-
24- In particular, these are branches that have started mirroring but have
25- not yet succeeded. Failed branches are included.
26- """
27-
28 merge_queue = Attribute(
29 "The queue that contains the QUEUED proposals for this branch.")
30
31
32=== modified file 'lib/lp/code/model/branch.py'
33--- lib/lp/code/model/branch.py 2010-04-19 03:17:04 +0000
34+++ lib/lp/code/model/branch.py 2010-04-27 02:07:14 +0000
35@@ -427,18 +427,6 @@
36 store = Store.of(self)
37 return store.find(Branch, Branch.stacked_on == self)
38
39- def getStackedBranchesWithIncompleteMirrors(self):
40- """See `IBranch`."""
41- store = Store.of(self)
42- return store.find(
43- Branch, Branch.stacked_on == self,
44- # Have been started.
45- Branch.last_mirror_attempt != None,
46- # Either never successfully mirrored or started since the last
47- # successful mirror.
48- Or(Branch.last_mirrored == None,
49- Branch.last_mirror_attempt > Branch.last_mirrored))
50-
51 def getMergeQueue(self):
52 """See `IBranch`."""
53 return BranchMergeProposal.select("""
54
55=== modified file 'lib/lp/code/model/branchpuller.py'
56--- lib/lp/code/model/branchpuller.py 2010-02-24 02:43:52 +0000
57+++ lib/lp/code/model/branchpuller.py 2010-04-27 02:07:14 +0000
58@@ -31,8 +31,7 @@
59 def acquireBranchToPull(self, *branch_types):
60 """See `IBranchPuller`."""
61 if not branch_types:
62- branch_types = (
63- BranchType.HOSTED, BranchType.MIRRORED, BranchType.IMPORTED)
64+ branch_types = (BranchType.MIRRORED, BranchType.IMPORTED)
65 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
66 branch = store.find(
67 Branch,
68
69=== modified file 'lib/lp/code/model/directbranchcommit.py'
70--- lib/lp/code/model/directbranchcommit.py 2010-01-21 22:10:38 +0000
71+++ lib/lp/code/model/directbranchcommit.py 2010-04-27 02:07:14 +0000
72@@ -172,7 +172,6 @@
73 if self.to_mirror:
74 return
75 assert self.is_locked, "Getting revision on un-locked branch."
76- last_revision = None
77 last_revision = self.bzrbranch.last_revision()
78 if last_revision != self.last_scanned_id:
79 raise ConcurrentUpdateError(
80@@ -189,7 +188,6 @@
81 assert self.is_open, "Committing closed DirectBranchCommit."
82 assert self.is_locked, "Not locked at commit time."
83
84- builder = None
85 try:
86 self._checkForRace()
87
88
89=== modified file 'lib/lp/code/model/tests/test_branch.py'
90--- lib/lp/code/model/tests/test_branch.py 2010-04-19 03:36:27 +0000
91+++ lib/lp/code/model/tests/test_branch.py 2010-04-27 02:07:14 +0000
92@@ -1292,79 +1292,6 @@
93 self.assertEqual(
94 set([stacked_a, stacked_b]), set(branch.getStackedBranches()))
95
96- def testStackedBranchesIncompleteMirrorsNoBranches(self):
97- # some_branch.getStackedBranchesWithIncompleteMirrors does not include
98- # stacked branches that haven't been mirrored at all.
99- branch = self.factory.makeAnyBranch()
100- self.factory.makeAnyBranch(stacked_on=branch)
101- self.assertEqual(
102- set(), set(branch.getStackedBranchesWithIncompleteMirrors()))
103-
104- def testStackedBranchesIncompleteMirrors(self):
105- # some_branch.getStackedBranchesWithIncompleteMirrors returns branches
106- # stacked on some_branch that had their mirrors started but not
107- # finished.
108- branch = self.factory.makeAnyBranch()
109- stacked_a = self.factory.makeAnyBranch(stacked_on=branch)
110- stacked_a.startMirroring()
111- self.assertEqual(
112- set([stacked_a]),
113- set(branch.getStackedBranchesWithIncompleteMirrors()))
114-
115- def testStackedBranchesIncompleteMirrorsNotStacked(self):
116- # some_branch.getStackedBranchesWithIncompleteMirrors does not include
117- # branches with incomplete mirrors that are not stacked on
118- # some_branch.
119- branch = self.factory.makeAnyBranch()
120- not_stacked = self.factory.makeAnyBranch()
121- not_stacked.startMirroring()
122- self.assertEqual(
123- set(), set(branch.getStackedBranchesWithIncompleteMirrors()))
124-
125- def testStackedBranchesCompleteMirrors(self):
126- # some_branch.getStackedBranchesWithIncompleteMirrors does not include
127- # branches that have been successfully mirrored.
128- branch = self.factory.makeAnyBranch()
129- stacked_a = self.factory.makeAnyBranch(stacked_on=branch)
130- stacked_a.startMirroring()
131- stacked_a.mirrorComplete(self.factory.getUniqueString())
132- self.assertEqual(
133- set(), set(branch.getStackedBranchesWithIncompleteMirrors()))
134-
135- def testStackedBranchesFailedMirrors(self):
136- # some_branch.getStackedBranchesWithIncompleteMirrors includes
137- # branches that failed to mirror. This is not directly desired, but is
138- # a consequence of wanting to include branches that have started,
139- # failed, then started again.
140- branch = self.factory.makeAnyBranch()
141- stacked_a = self.factory.makeAnyBranch(stacked_on=branch)
142- stacked_a.startMirroring()
143- stacked_a.mirrorFailed(self.factory.getUniqueString())
144- self.assertEqual(
145- set([stacked_a]),
146- set(branch.getStackedBranchesWithIncompleteMirrors()))
147-
148- def testStackedBranchesFailedThenStartedMirrors(self):
149- # some_branch.getStackedBranchesWithIncompleteMirrors includes
150- # branches that had a failed mirror but have since been started.
151- branch = self.factory.makeAnyBranch()
152- stacked_a = self.factory.makeAnyBranch(stacked_on=branch)
153- stacked_a.startMirroring()
154- stacked_a.mirrorFailed(self.factory.getUniqueString())
155- stacked_a.startMirroring()
156- self.assertEqual(
157- set([stacked_a]),
158- set(branch.getStackedBranchesWithIncompleteMirrors()))
159-
160- def testStackedBranchesMirrorRequested(self):
161- # some_branch.getStackedBranchesWithIncompleteMirrors does not include
162- # branches that have only had a mirror requested.
163- branch = self.factory.makeAnyBranch()
164- stacked_a = self.factory.makeAnyBranch(stacked_on=branch)
165- stacked_a.requestMirror()
166- self.assertEqual(
167- set(), set(branch.getStackedBranchesWithIncompleteMirrors()))
168-
169
170 class BranchAddLandingTarget(TestCaseWithFactory):
171 """Exercise all the code paths for adding a landing target."""
172
173=== modified file 'lib/lp/code/model/tests/test_branchpuller.py'
174--- lib/lp/code/model/tests/test_branchpuller.py 2010-02-24 08:59:31 +0000
175+++ lib/lp/code/model/tests/test_branchpuller.py 2010-04-27 02:07:14 +0000
176@@ -21,11 +21,12 @@
177 from lp.testing import TestCaseWithFactory, login_person
178
179
180-class TestMirroringForHostedBranches(TestCaseWithFactory):
181+class TestMirroringForImportedBranches(TestCaseWithFactory):
182 """Tests for mirroring methods of a branch."""
183
184 layer = DatabaseFunctionalLayer
185- branch_type = BranchType.HOSTED
186+
187+ branch_type = BranchType.IMPORTED
188
189 def setUp(self):
190 TestCaseWithFactory.setUp(self)
191@@ -98,12 +99,12 @@
192 self.assertEqual(None, branch.next_mirror_time)
193
194
195-class TestMirroringForMirroredBranches(TestMirroringForHostedBranches):
196+class TestMirroringForMirroredBranches(TestMirroringForImportedBranches):
197
198 branch_type = BranchType.MIRRORED
199
200 def setUp(self):
201- TestMirroringForHostedBranches.setUp(self)
202+ TestMirroringForImportedBranches.setUp(self)
203 branch_puller = getUtility(IBranchPuller)
204 self.increment = branch_puller.MIRROR_TIME_INCREMENT
205 self.max_failures = branch_puller.MAXIMUM_MIRROR_FAILURES
206@@ -156,11 +157,6 @@
207 self.assertEqual(0, branch.mirror_failures)
208
209
210-class TestMirroringForImportedBranches(TestMirroringForHostedBranches):
211-
212- branch_type = BranchType.IMPORTED
213-
214-
215 class AcquireBranchToPullTests:
216 """Tests for acquiring branches to pull.
217
218@@ -199,7 +195,7 @@
219 def test_simple(self):
220 # If there is one branch that needs mirroring, acquireBranchToPull
221 # returns that.
222- branch = self.factory.makeAnyBranch()
223+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
224 branch.requestMirror()
225 self.assertBranchIsAquired(branch)
226
227@@ -207,7 +203,7 @@
228 # On a few occasions a branch type that is mirrored has been
229 # converted, with non-NULL next_mirror_time, to a remote branch, which
230 # is not mirrored. These branches should not be returned.
231- branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
232+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
233 branch.requestMirror()
234 removeSecurityProxy(branch).branch_type = BranchType.REMOTE
235 self.assertNoBranchIsAquired()
236@@ -215,13 +211,14 @@
237 def test_private(self):
238 # If there is a private branch that needs mirroring,
239 # acquireBranchToPull returns that.
240- branch = self.factory.makeAnyBranch(private=True)
241+ branch = self.factory.makeAnyBranch(
242+ branch_type=BranchType.MIRRORED, private=True)
243 removeSecurityProxy(branch).requestMirror()
244 self.assertBranchIsAquired(branch)
245
246 def test_no_inprogress(self):
247 # If a branch is being mirrored, it is not returned.
248- branch = self.factory.makeAnyBranch()
249+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
250 branch.requestMirror()
251 self.startMirroring(branch)
252 self.assertNoBranchIsAquired()
253@@ -229,44 +226,41 @@
254 def test_first_requested_returned(self):
255 # If two branches are to be mirrored, the one that was requested first
256 # is returned.
257- first_branch = self.factory.makeAnyBranch()
258+ first_branch = self.factory.makeAnyBranch(
259+ branch_type=BranchType.MIRRORED)
260 # You can only request a mirror now, so to pretend that we requested
261 # it some time ago, we cheat with removeSecurityProxy().
262 first_branch.requestMirror()
263 naked_first_branch = removeSecurityProxy(first_branch)
264 naked_first_branch.next_mirror_time -= timedelta(seconds=100)
265- second_branch = self.factory.makeAnyBranch()
266+ second_branch = self.factory.makeAnyBranch(
267+ branch_type=BranchType.MIRRORED)
268 second_branch.requestMirror()
269 naked_second_branch = removeSecurityProxy(second_branch)
270 naked_second_branch.next_mirror_time -= timedelta(seconds=50)
271 self.assertBranchIsAquired(naked_first_branch)
272
273- def test_type_filter_hosted_returns_hosted(self):
274- branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
275- branch.requestMirror()
276- self.assertBranchIsAquired(branch, BranchType.HOSTED)
277-
278- def test_type_filter_hosted_does_not_return_mirrored(self):
279- branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
280- branch.requestMirror()
281- self.assertNoBranchIsAquired(BranchType.HOSTED)
282-
283- def test_type_filter_mirrored_does_not_return_hosted(self):
284- branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
285- branch.requestMirror()
286- self.assertNoBranchIsAquired(BranchType.MIRRORED)
287-
288- def test_type_filter_hosted_imported_returns_hosted(self):
289- branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
290+ def test_type_filter_mirrrored_returns_mirrored(self):
291+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
292+ branch.requestMirror()
293+ self.assertBranchIsAquired(branch, BranchType.MIRRORED)
294+
295+ def test_type_filter_imported_does_not_return_mirrored(self):
296+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
297+ branch.requestMirror()
298+ self.assertNoBranchIsAquired(BranchType.IMPORTED)
299+
300+ def test_type_filter_mirrored_imported_returns_mirrored(self):
301+ branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
302 branch.requestMirror()
303 self.assertBranchIsAquired(
304- branch, BranchType.HOSTED, BranchType.IMPORTED)
305+ branch, BranchType.MIRRORED, BranchType.IMPORTED)
306
307- def test_type_filter_hosted_imported_returns_imported(self):
308+ def test_type_filter_mirrored_imported_returns_imported(self):
309 branch = self.factory.makeAnyBranch(branch_type=BranchType.IMPORTED)
310 branch.requestMirror()
311 self.assertBranchIsAquired(
312- branch, BranchType.HOSTED, BranchType.IMPORTED)
313+ branch, BranchType.MIRRORED, BranchType.IMPORTED)
314
315
316 class TestAcquireBranchToPullDirectly(TestCaseWithFactory,
317
318=== modified file 'lib/lp/code/xmlrpc/codehosting.py'
319--- lib/lp/code/xmlrpc/codehosting.py 2010-04-27 02:06:51 +0000
320+++ lib/lp/code/xmlrpc/codehosting.py 2010-04-27 02:07:14 +0000
321@@ -93,9 +93,6 @@
322 # See comment in startMirroring.
323 branch = removeSecurityProxy(branch)
324 branch.mirrorComplete(last_revision_id)
325- branches = branch.getStackedBranchesWithIncompleteMirrors()
326- for stacked_branch in branches:
327- stacked_branch.requestMirror()
328 return True
329
330 def mirrorFailed(self, branch_id, reason):
331
332=== modified file 'lib/lp/code/xmlrpc/tests/test_codehosting.py'
333--- lib/lp/code/xmlrpc/tests/test_codehosting.py 2010-04-27 02:06:51 +0000
334+++ lib/lp/code/xmlrpc/tests/test_codehosting.py 2010-04-27 02:07:14 +0000
335@@ -272,58 +272,6 @@
336
337 self.assertIs(None, branch.next_mirror_time)
338
339- def test_mirrorComplete_requests_mirror_for_incomplete_stacked(self):
340- # After successfully mirroring a branch on which others are stacked,
341- # any stacked branches with incomplete mirrors should have a mirror
342- # requested. This prevents them from being trapped in a failed state.
343- # See bug 261334.
344- branch = self.factory.makeAnyBranch()
345- stacked_branch = self.factory.makeAnyBranch(stacked_on=branch)
346-
347- # Note that no mirror is requested.
348- self.assertIs(None, stacked_branch.next_mirror_time)
349-
350- self.storage.startMirroring(stacked_branch.id)
351- self.storage.startMirroring(branch.id)
352- self.storage.mirrorComplete(branch.id, self.factory.getUniqueString())
353- self.assertSqlAttributeEqualsDate(
354- stacked_branch, 'next_mirror_time', UTC_NOW)
355-
356- def test_mirrorCompleteRequestsMirrorForIncompleteStackedOnPrivate(self):
357- # After successfully mirroring a *private* branch on which others are
358- # stacked, any stacked branches with incomplete mirrors have a mirror
359- # requested. See bug 261334.
360- branch = removeSecurityProxy(
361- self.factory.makeAnyBranch(private=True))
362- stacked_branch = removeSecurityProxy(
363- self.factory.makeAnyBranch(stacked_on=branch, private=True))
364-
365- # Note that no mirror is requested.
366- self.assertIs(None, stacked_branch.next_mirror_time)
367-
368- self.storage.startMirroring(stacked_branch.id)
369- self.storage.startMirroring(branch.id)
370- self.storage.mirrorComplete(branch.id, self.factory.getUniqueString())
371- self.assertSqlAttributeEqualsDate(
372- stacked_branch, 'next_mirror_time', UTC_NOW)
373-
374- def test_mirrorCompletePrivateStackedOnPublic(self):
375- # After successfully mirroring a *public* branch on which *private*
376- # branche are stacked, any stacked branches with incomplete mirrors
377- # have a mirror requested. See bug 261334.
378- branch = self.factory.makeAnyBranch()
379- stacked_branch = removeSecurityProxy(
380- self.factory.makeAnyBranch(stacked_on=branch, private=True))
381-
382- # Note that no mirror is requested.
383- self.assertIs(None, stacked_branch.next_mirror_time)
384-
385- self.storage.startMirroring(stacked_branch.id)
386- self.storage.startMirroring(branch.id)
387- self.storage.mirrorComplete(branch.id, self.factory.getUniqueString())
388- self.assertSqlAttributeEqualsDate(
389- stacked_branch, 'next_mirror_time', UTC_NOW)
390-
391 def test_recordSuccess(self):
392 # recordSuccess must insert the given data into ScriptActivity.
393 started = datetime.datetime(2007, 07, 05, 19, 32, 1, tzinfo=UTC)
394@@ -434,13 +382,6 @@
395 """See `AcquireBranchToPullTests`."""
396 self.storage.startMirroring(branch.id)
397
398- def test_branch_type_returned_hosted(self):
399- branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
400- branch.requestMirror()
401- pull_info = self.storage.acquireBranchToPull(())
402- _, _, _, _, branch_type = pull_info
403- self.assertEqual('HOSTED', branch_type)
404-
405 def test_branch_type_returned_mirrored(self):
406 branch = self.factory.makeAnyBranch(branch_type=BranchType.MIRRORED)
407 branch.requestMirror()
408@@ -456,7 +397,8 @@
409 self.assertEqual('IMPORTED', branch_type)
410
411 def test_default_stacked_on_branch_returned(self):
412- branch = self.factory.makeProductBranch()
413+ branch = self.factory.makeProductBranch(
414+ branch_type=BranchType.MIRRORED)
415 self.factory.enableDefaultStackingForProduct(branch.product)
416 branch.requestMirror()
417 pull_info = self.storage.acquireBranchToPull(())
418
419=== modified file 'lib/lp/codehosting/puller/scheduler.py'
420--- lib/lp/codehosting/puller/scheduler.py 2010-03-19 10:43:51 +0000
421+++ lib/lp/codehosting/puller/scheduler.py 2010-04-27 02:07:14 +0000
422@@ -233,9 +233,6 @@
423 self.resetTimeout()
424 self.runNotification(self.listener.startMirroring)
425
426- def do_mirrorDeferred(self):
427- self.reported_mirror_finished = True
428-
429 def do_mirrorSucceeded(self, revid_before, revid_after):
430 def mirrorSucceeded():
431 d = defer.maybeDeferred(
432@@ -267,10 +264,7 @@
433 generated by that process.
434 """
435
436- path_to_script = os.path.join(
437- os.path.dirname(
438- os.path.dirname(os.path.dirname(canonical.__file__))),
439- 'scripts/mirror-branch.py')
440+ path_to_script = os.path.join(config.root, 'scripts/mirror-branch.py')
441 protocol_class = PullerMonitorProtocol
442
443 def __init__(self, branch_id, source_url, unique_name, branch_type_name,
444@@ -282,8 +276,8 @@
445 :param source_url: The location from which the branch is to be
446 mirrored.
447 :param unique_name: The unique name of the branch to be mirrored.
448- :param branch_type: The BranchType of the branch to be mirrored (e.g.
449- BranchType.HOSTED).
450+ :param branch_type_name: The name of the BranchType of the branch to
451+ be mirrored (e.g. 'MIRRORED').
452 :param default_stacked_on_url: The default stacked-on URL for the
453 product that the branch is in. '' implies that there is no such
454 default.
455@@ -297,7 +291,7 @@
456 """
457 self.branch_id = branch_id
458 self.source_url = source_url.strip()
459- self.destination_url = 'lp-mirrored:///%s' % (unique_name,)
460+ self.destination_url = 'lp-internal:///%s' % (unique_name,)
461 self.unique_name = unique_name
462 self.branch_type_name = branch_type_name
463 self.default_stacked_on_url = default_stacked_on_url
464
465=== modified file 'lib/lp/codehosting/puller/tests/__init__.py'
466--- lib/lp/codehosting/puller/tests/__init__.py 2010-04-14 03:07:42 +0000
467+++ lib/lp/codehosting/puller/tests/__init__.py 2010-04-27 02:07:14 +0000
468@@ -9,7 +9,7 @@
469 import shutil
470 from StringIO import StringIO
471
472-from bzrlib.tests import TestCaseWithTransport
473+from bzrlib.tests import HttpServer, TestCaseWithTransport
474 from bzrlib import urlutils
475
476 from lp.codehosting.vfs import branch_id_to_path
477@@ -151,3 +151,18 @@
478 retcode=None)
479 # We want to be sure that a new branch was indeed created.
480 self.assertEqual("Created new branch.\n", err)
481+
482+ def serveOverHTTP(self):
483+ """Serve the current directory over HTTP, returning the server URL."""
484+ http_server = HttpServer()
485+ http_server.start_server()
486+ # Join cleanup added before the tearDown so the tearDown is executed
487+ # first as this tells the thread to die. We then join explicitly as
488+ # the HttpServer.tearDown does not join. There is a check in the
489+ # BaseLayer to make sure that threads are not left behind by the
490+ # tests, and the default behaviour of the HttpServer is to use daemon
491+ # threads and let the garbage collector get them, however this causes
492+ # issues with the test runner.
493+ self.addCleanup(http_server._http_thread.join)
494+ self.addCleanup(http_server.stop_server)
495+ return http_server.get_url().rstrip('/')
496
497=== modified file 'lib/lp/codehosting/puller/tests/test_acceptance.py'
498--- lib/lp/codehosting/puller/tests/test_acceptance.py 2010-04-15 01:30:39 +0000
499+++ lib/lp/codehosting/puller/tests/test_acceptance.py 2010-04-27 02:07:14 +0000
500@@ -7,33 +7,38 @@
501 __all__ = []
502
503
504+import datetime
505 import os
506 from subprocess import PIPE, Popen
507 import unittest
508
509+import pytz
510+
511 import transaction
512
513 from bzrlib.branch import Branch
514-from bzrlib.bzrdir import BzrDir, format_registry
515-from bzrlib.config import TransportConfig
516+from bzrlib.bzrdir import BzrDir
517 from bzrlib import errors
518-from bzrlib.tests import HttpServer
519 from bzrlib.transport import get_transport
520 from bzrlib.upgrade import upgrade
521-from bzrlib.urlutils import join as urljoin, local_path_from_url
522+from bzrlib.urlutils import (
523+ join as urljoin, local_path_from_url, local_path_to_url)
524+from bzrlib.workingtree import WorkingTree
525
526 from zope.component import getUtility
527 from zope.security.proxy import removeSecurityProxy
528
529 from lp.code.enums import BranchType
530-from lp.codehosting.vfs import get_lp_server
531+from lp.code.interfaces.branchtarget import IBranchTarget
532+from lp.codehosting.vfs import branch_id_to_path, get_lp_server
533 from lp.codehosting.puller.tests import PullerBranchTestCase
534+from lp.codehosting.tests.helpers import LoomTestMixin
535 from canonical.config import config
536 from canonical.launchpad.interfaces import IScriptActivitySet
537 from canonical.testing import ZopelessAppServerLayer
538
539
540-class TestBranchPuller(PullerBranchTestCase):
541+class TestBranchPuller(PullerBranchTestCase, LoomTestMixin):
542 """Integration tests for the branch puller.
543
544 These tests actually run the supermirror-pull.py script. Instead of
545@@ -47,12 +52,11 @@
546 PullerBranchTestCase.setUp(self)
547 self._puller_script = os.path.join(
548 config.root, 'cronscripts', 'supermirror-pull.py')
549- self.makeCleanDirectory(config.codehosting.hosted_branches_root)
550 self.makeCleanDirectory(config.codehosting.mirrored_branches_root)
551 self.makeCleanDirectory(
552 local_path_from_url(config.launchpad.bzr_imports_root_url))
553
554- def assertMirrored(self, db_branch, source_branch=None,
555+ def assertMirrored(self, db_branch, source_branch,
556 accessing_user=None):
557 """Assert that 'db_branch' was mirrored succesfully.
558
559@@ -69,8 +73,6 @@
560 supplied create a fresh user for this -- but this won't work for a
561 private branch.
562 """
563- if source_branch is None:
564- source_branch = self.openBranchAsUser(db_branch, db_branch.owner)
565 if accessing_user is None:
566 accessing_user = self.factory.makePerson()
567 transaction.commit()
568@@ -133,21 +135,6 @@
569 retcode, output, error = self.runSubprocess(command)
570 return command, retcode, output, error
571
572- def serveOverHTTP(self):
573- """Serve the current directory over HTTP, returning the server URL."""
574- http_server = HttpServer()
575- http_server.start_server()
576- # Join cleanup added before the tearDown so the tearDown is executed
577- # first as this tells the thread to die. We then join explicitly as
578- # the HttpServer.tearDown does not join. There is a check in the
579- # BaseLayer to make sure that threads are not left behind by the
580- # tests, and the default behaviour of the HttpServer is to use daemon
581- # threads and let the garbage collector get them, however this causes
582- # issues with the test runner.
583- self.addCleanup(http_server._http_thread.join)
584- self.addCleanup(http_server.stop_server)
585- return http_server.get_url().rstrip('/')
586-
587 def getLPServerForUser(self, user):
588 """Construct a LaunchpadServer that serves branches as seen by `user`.
589
590@@ -173,28 +160,6 @@
591 lp_server = self.getLPServerForUser(user)
592 return Branch.open(lp_server.get_url() + db_branch.unique_name)
593
594- def pushBranch(self, db_branch, tree=None, format=None):
595- """Push a Bazaar branch to db_branch.
596-
597- This method pushes the branch of the supplied tree (or an empty branch
598- containing one revision if no tree is suppplied) to the location
599- represented by the database branch 'db_branch'.
600- """
601- if tree is None:
602- tree = self.make_branch_and_tree(
603- self.factory.getUniqueString(), format=format)
604- tree.commit('rev1')
605- lp_server = self.getLPServerForUser(db_branch.owner)
606- dest_transport = get_transport(
607- lp_server.get_url() + db_branch.unique_name)
608- try:
609- dir_to = BzrDir.open_from_transport(dest_transport)
610- except errors.NotBranchError:
611- # create new branch
612- tree.branch.bzrdir.clone_on_transport(dest_transport)
613- else:
614- tree.branch.push(dir_to.open_branch())
615-
616 def setUpMirroredBranch(self, db_branch, format=None):
617 """Make a tree in the cwd and serve it over HTTP, returning the URL.
618 """
619@@ -204,105 +169,72 @@
620 db_branch.requestMirror()
621 return tree
622
623- def test_mirror_hosted_branch(self):
624- # Run the puller on a populated hosted branch pull queue.
625- db_branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
626- transaction.commit()
627- self.pushBranch(db_branch)
628- command, retcode, output, error = self.runPuller()
629- self.assertRanSuccessfully(command, retcode, output, error)
630- self.assertMirrored(db_branch)
631-
632- def test_remirror_hosted_branch(self):
633- # When the format of a branch changes, we completely remirror it.
634- # First we push up and mirror the branch in one format.
635- db_branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
636- transaction.commit()
637- pack_tree = self.make_branch_and_tree('pack', format='pack-0.92')
638- self.pushBranch(db_branch, tree=pack_tree)
639- command, retcode, output, error = self.runPuller()
640- self.assertRanSuccessfully(command, retcode, output, error)
641- self.assertMirrored(db_branch)
642- # Then we upgrade the to a different format and ask for it to be
643- # mirrored again.
644- upgrade(self.getHostedPath(db_branch), format_registry.get('1.6')())
645+ def test_mirror_mirrored_branch(self):
646+ # Run the puller with a mirrored branch ready to be pulled.
647+ db_branch = self.factory.makeAnyBranch(
648+ branch_type=BranchType.MIRRORED)
649+ tree = self.setUpMirroredBranch(db_branch)
650+ transaction.commit()
651+ command, retcode, output, error = self.runPuller()
652+ self.assertRanSuccessfully(command, retcode, output, error)
653+ self.assertMirrored(db_branch, source_branch=tree.branch)
654+
655+ def test_mirror_mirrored_loom_branch(self):
656+ # Run the puller with a mirrored loom branch ready to be pulled.
657+ db_branch = self.factory.makeAnyBranch(
658+ branch_type=BranchType.MIRRORED)
659+ tree = self.setUpMirroredBranch(db_branch)
660+ self.loomify(tree.branch)
661+ transaction.commit()
662+ command, retcode, output, error = self.runPuller()
663+ self.assertRanSuccessfully(command, retcode, output, error)
664+ self.assertMirrored(db_branch, source_branch=tree.branch)
665+
666+ def test_format_change(self):
667+ # When the format of a mirrored branch changes, the puller remirrors
668+ # the branch into the new format.
669+ db_branch = self.factory.makeAnyBranch(
670+ branch_type=BranchType.MIRRORED)
671+ tree = self.setUpMirroredBranch(db_branch, format='pack-0.92')
672+ transaction.commit()
673+ command, retcode, output, error = self.runPuller()
674+ self.assertRanSuccessfully(command, retcode, output, error)
675+ self.assertMirrored(db_branch, source_branch=tree.branch)
676 transaction.begin()
677 db_branch.requestMirror()
678- transaction.commit()
679- command, retcode, output, error = self.runPuller()
680- self.assertRanSuccessfully(command, retcode, output, error)
681- self.assertMirrored(db_branch)
682-
683- def test_mirror_hosted_loom_branch(self):
684- # Run the puller over a branch with looms enabled.
685- db_branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
686- transaction.commit()
687- loom_tree = self.makeLoomBranchAndTree('loom')
688- self.pushBranch(db_branch, tree=loom_tree)
689- command, retcode, output, error = self.runPuller()
690- self.assertRanSuccessfully(command, retcode, output, error)
691- self.assertMirrored(db_branch)
692-
693- def test_mirror_private_branch(self):
694- # Run the puller with a private branch in the queue.
695- db_branch = self.factory.makeAnyBranch(
696- branch_type=BranchType.HOSTED, private=True)
697- accessing_user = self.factory.makePerson()
698- self.factory.makeBranchSubscription(
699- branch=db_branch, person=accessing_user)
700- transaction.commit()
701- self.pushBranch(db_branch)
702- command, retcode, output, error = self.runPuller()
703- self.assertRanSuccessfully(command, retcode, output, error)
704- self.assertMirrored(db_branch, accessing_user=accessing_user)
705-
706- def test_mirror_mirrored_branch(self):
707- # Run the puller on a populated mirrored branch pull queue.
708- db_branch = self.factory.makeAnyBranch(
709- branch_type=BranchType.MIRRORED)
710- tree = self.setUpMirroredBranch(db_branch)
711+ upgrade(tree.basedir)
712+ tree = WorkingTree.open(tree.basedir)
713 transaction.commit()
714 command, retcode, output, error = self.runPuller()
715 self.assertRanSuccessfully(command, retcode, output, error)
716 self.assertMirrored(db_branch, source_branch=tree.branch)
717
718- def _makeDefaultStackedOnBranch(self, private=False,
719- branch_type=BranchType.HOSTED):
720+ def _makeDefaultStackedOnBranch(self, private=False):
721 """Make a default stacked-on branch.
722
723- This creates a database branch on a product that allows default
724- stacking, makes it the default stacked-on branch for that product,
725- creates a Bazaar branch for it and pulls it over into the mirrored
726- area.
727+ This creates a database product branch, makes it the default
728+ stacked-on branch for its product and creates a Bazaar branch for it.
729
730+ :param private: Whether the created branch should be private or not
731+ (defaults to not).
732 :return: `IBranch`.
733 """
734 # Make the branch.
735 product = self.factory.makeProduct()
736 default_branch = self.factory.makeProductBranch(
737- product=product, private=private, name='trunk',
738- branch_type=branch_type)
739+ product=product, private=private)
740+ default_branch.last_mirrored = datetime.datetime.now(pytz.UTC)
741 # Make it the default stacked-on branch.
742 series = removeSecurityProxy(product.development_focus)
743 series.branch = default_branch
744- # Arrange for it to be pulled.
745- if branch_type == BranchType.HOSTED:
746- transaction.commit()
747- # For hosted branches, we just push it into the hosted area via
748- # the codehosting vfs.
749- self.pushBranch(default_branch)
750- elif branch_type == BranchType.MIRRORED:
751- # For mirrored branches, we serve the branch over HTTP, point the
752- # database branch at this HTTP server and call requestMirror()
753- self.setUpMirroredBranch(default_branch)
754- transaction.commit()
755- else:
756- raise AssertionError(
757- "don't know how to make a %s default branch"
758- % branch_type.TITLE)
759- # Pull it.
760- command, retcode, output, error = self.runPuller()
761- self.assertRanSuccessfully(command, retcode, output, error)
762+ self.assertEqual(
763+ default_branch, IBranchTarget(product).default_stacked_on_branch)
764+ branch_location = urljoin(
765+ local_path_to_url(config.codehosting.mirrored_branches_root),
766+ branch_id_to_path(default_branch.id))
767+ get_transport(branch_location).create_prefix()
768+ BzrDir.create_branch_convenience(branch_location)
769+ transaction.commit()
770 return default_branch
771
772 def test_stack_mirrored_branch(self):
773@@ -321,57 +253,6 @@
774 '/' + default_branch.unique_name,
775 mirrored_branch.get_stacked_on_url())
776
777- def test_hosted_branch_stacked_on_mirrored_branch(self):
778- # If a hosted branch is stacked on a mirrored branch, the puller opens
779- # the hosted branch stacked on the copy of the branch in the mirrored
780- # area, rather than trying to open it stacked on the non-existent
781- # version of the branch in the hosted area.
782- default_branch = self._makeDefaultStackedOnBranch(
783- branch_type=BranchType.MIRRORED)
784- db_branch = self.factory.makeProductBranch(
785- branch_type=BranchType.HOSTED, product=default_branch.product)
786- transaction.commit()
787- self.pushBranch(db_branch)
788- command, retcode, output, error = self.runPuller()
789- self.assertRanSuccessfully(command, retcode, output, error)
790- mirrored_branch = self.assertMirrored(db_branch)
791- self.assertEqual(
792- '/' + default_branch.unique_name,
793- mirrored_branch.get_stacked_on_url())
794-
795- def test_manual_stacking(self):
796- # If the user manually stacks on a Launchpad branch, the branch.conf
797- # file of the resulting branch will contain the full URL of the
798- # manually selected branch. The puller still manages to open the
799- # branch and sets the stacking information of the branch in the
800- # mirrored area to be the most compatible "/" + unique_name form. We
801- # have to cheat rather to test this because the full URLs don't work
802- # in the environment this test is run in, so we set the stacking URL
803- # in the hosted area directory after we've pushed it and then clear it
804- # again so that assertMirrored can work. The test is still valid
805- # though, as the paths are as they should be when the puller is run.
806- default_branch = self._makeDefaultStackedOnBranch()
807- db_branch = self.factory.makeProductBranch(
808- branch_type=BranchType.HOSTED, product=default_branch.product)
809- transaction.commit()
810- self.pushBranch(db_branch)
811- # Because Bazaar can't access branches over bzr+ssh in this test, we
812- # cheat and set the stacking information directly.
813- branch_config = TransportConfig(
814- get_transport(self.getHostedPath(db_branch)), 'branch.conf')
815- branch_config.set_option(
816- 'stacked_on_location',
817- 'bzr+ssh://bazaar.launchpad.dev/' + default_branch.unique_name)
818- command, retcode, output, error = self.runPuller()
819- self.assertRanSuccessfully(command, retcode, output, error)
820- # We clear the stacking information again here so that assertMirrored
821- # can open the branch in the hosted area.
822- branch_config.set_option('stacked_on_location', '')
823- mirrored_branch = self.assertMirrored(db_branch)
824- self.assertEqual(
825- '/' + default_branch.unique_name,
826- mirrored_branch.get_stacked_on_url())
827-
828 def test_stack_mirrored_branch_onto_private(self):
829 # If the default stacked-on branch is private then mirrored branches
830 # aren't stacked when they are mirrored.
831@@ -403,6 +284,8 @@
832 tree = branch.bzrdir.open_workingtree()
833 tree.commit('rev1')
834
835+ transaction.commit()
836+
837 # Run the puller.
838 command, retcode, output, error = self.runPuller()
839 self.assertRanSuccessfully(command, retcode, output, error)
840@@ -417,19 +300,22 @@
841 def test_type_filtering(self):
842 # When run with --branch-type arguments, the puller only mirrors those
843 # branches of the specified types.
844- hosted_branch = self.factory.makeAnyBranch(
845- branch_type=BranchType.HOSTED)
846+ imported_branch = self.factory.makeAnyBranch(
847+ branch_type=BranchType.IMPORTED)
848+ imported_branch.requestMirror()
849 mirrored_branch = self.factory.makeAnyBranch(
850 branch_type=BranchType.MIRRORED)
851 mirrored_branch.requestMirror()
852+ tree = self.setUpMirroredBranch(mirrored_branch)
853+
854 transaction.commit()
855- self.pushBranch(hosted_branch)
856+
857 command, retcode, output, error = self.runPuller(
858- '--branch-type', 'HOSTED')
859+ '--branch-type', 'MIRRORED')
860 self.assertRanSuccessfully(command, retcode, output, error)
861- self.assertMirrored(hosted_branch)
862+ self.assertMirrored(mirrored_branch, source_branch=tree.branch)
863 self.assertIsNot(
864- None, mirrored_branch.next_mirror_time)
865+ None, imported_branch.next_mirror_time)
866
867 def test_records_script_activity(self):
868 # A record gets created in the ScriptActivity table.
869
870=== modified file 'lib/lp/codehosting/puller/tests/test_errors.py'
871--- lib/lp/codehosting/puller/tests/test_errors.py 2009-08-27 02:00:29 +0000
872+++ lib/lp/codehosting/puller/tests/test_errors.py 2010-04-27 02:07:14 +0000
873@@ -18,7 +18,7 @@
874
875 from lp.codehosting.puller.worker import (
876 BranchMirrorer, BranchReferenceForbidden, BranchLoopError, PullerWorker,
877- PullerWorkerProtocol, StackedOnBranchNotFound)
878+ PullerWorkerProtocol)
879 from lp.codehosting.vfs.branchfs import (
880 BadUrlLaunchpad, BadUrlScheme, BadUrlSsh)
881 from lp.code.enums import BranchType
882@@ -112,14 +112,6 @@
883 branch_type=BranchType.HOSTED)
884 self.assertEqual(expected_msg, msg)
885
886- def testStackedOnBranchNotFound(self):
887- # If StackedOnBranchNotFound is raised then we send mirrorDeferred to
888- # the scheduler.
889- worker = self.makeRaisingWorker(StackedOnBranchNotFound())
890- worker.mirror()
891- self.assertEqual(
892- [('startMirroring',), ('mirrorDeferred',)], worker.protocol.calls)
893-
894 def testLocalURL(self):
895 # A file:// branch reference for a mirror branch must cause an error.
896 expected_msg = (
897
898=== modified file 'lib/lp/codehosting/puller/tests/test_scheduler.py'
899--- lib/lp/codehosting/puller/tests/test_scheduler.py 2010-03-19 10:43:51 +0000
900+++ lib/lp/codehosting/puller/tests/test_scheduler.py 2010-04-27 02:07:14 +0000
901@@ -15,6 +15,7 @@
902
903 from bzrlib.branch import Branch
904 from bzrlib.bzrdir import BzrDir
905+from bzrlib.urlutils import join as urljoin
906
907 from twisted.internet import defer, error, reactor
908 from twisted.protocols.basic import NetstringParseError
909@@ -100,10 +101,10 @@
910 manager.unlock()
911
912 def test_run_calls_acquireBranchToPull(self):
913- manager = self.makeJobScheduler(('HOSTED',))
914+ manager = self.makeJobScheduler(('MIRRORED',))
915 manager.run()
916 self.assertEqual(
917- [('acquireBranchToPull', ('HOSTED',))],
918+ [('acquireBranchToPull', ('MIRRORED',))],
919 manager.branch_puller_endpoint.calls)
920
921
922@@ -273,16 +274,6 @@
923 [('mirrorSucceeded', 'rev1', 'rev2')], self.listener.calls)
924 self.assertProtocolSuccess()
925
926- def test_mirrorDeferred(self):
927- # Receiving a mirrorDeferred message finishes mirroring and doesn't
928- # notify the listener.
929- self.protocol.do_startMirroring()
930- self.listener.calls = []
931- self.protocol.do_mirrorDeferred()
932- self.assertProtocolSuccess()
933- self.assertEqual(True, self.protocol.reported_mirror_finished)
934- self.assertEqual([], self.listener.calls)
935-
936 def test_mirrorFailed(self):
937 """Receiving a mirrorFailed message notifies the listener."""
938 self.protocol.do_startMirroring()
939@@ -473,15 +464,6 @@
940
941 return deferred.addCallback(checkSetStackedOn)
942
943- def test_setStackedOnBranchNotFound(self):
944- stacked_on_location = 'raise-branch-not-found'
945- deferred = self.eventHandler.setStackedOn(stacked_on_location)
946-
947- def checkSetStackedOn(ignored):
948- self.assertEqual([], self.status_client.calls)
949-
950- return deferred.addCallback(checkSetStackedOn)
951-
952 def test_mirrorComplete(self):
953 arbitrary_revision_ids = ('rev1', 'rev2')
954 deferred = defer.maybeDeferred(self.eventHandler.startMirroring)
955@@ -652,15 +634,14 @@
956 def setUp(self):
957 TrialTestCase.setUp(self)
958 PullerBranchTestCase.setUp(self)
959- self.makeCleanDirectory(config.codehosting.hosted_branches_root)
960 self.makeCleanDirectory(config.codehosting.mirrored_branches_root)
961+ self.bzr_tree = self.make_branch_and_tree('src-branch')
962+ url = urljoin(self.serveOverHTTP(), 'src-branch')
963+ self.bzr_tree.commit('rev1')
964 branch_id = self.factory.makeAnyBranch(
965- branch_type=BranchType.HOSTED).id
966+ branch_type=BranchType.MIRRORED, url=url).id
967 self.layer.txn.commit()
968 self.db_branch = getUtility(IBranchLookup).get(branch_id)
969- self.bzr_tree = self.make_branch_and_tree('src-branch')
970- self.bzr_tree.commit('rev1')
971- self.pushToBranch(self.db_branch, self.bzr_tree)
972 self.client = FakePullerEndpointProxy()
973
974 def run(self, result):
975@@ -688,9 +669,8 @@
976 worker command line arguments, the destination branch and an
977 instance of PullerWorkerProtocol.
978 """
979- hosted_url = str('lp-hosted:///' + self.db_branch.unique_name)
980 puller_master = cls(
981- self.db_branch.id, hosted_url,
982+ self.db_branch.id, str(self.db_branch.url),
983 self.db_branch.unique_name[1:], self.db_branch.branch_type.name,
984 '', logging.getLogger(), self.client,
985 set([config.error_reports.oops_prefix]))
986@@ -714,7 +694,7 @@
987
988 def check_authserver_called(ignored):
989 self.assertEqual(
990- [('setStackedOn', 77, ''),
991+ [('setStackedOn', self.db_branch.id, ''),
992 ('mirrorComplete', self.db_branch.id, revision_id)],
993 self.client.calls)
994 return ignored
995
996=== modified file 'lib/lp/codehosting/puller/tests/test_worker.py'
997--- lib/lp/codehosting/puller/tests/test_worker.py 2010-01-21 22:54:23 +0000
998+++ lib/lp/codehosting/puller/tests/test_worker.py 2010-04-27 02:07:14 +0000
999@@ -23,17 +23,15 @@
1000
1001 from lp.codehosting.puller.worker import (
1002 BranchLoopError, BranchMirrorer, BranchReferenceForbidden,
1003- PullerWorkerProtocol, StackedOnBranchNotFound,
1004- install_worker_ui_factory, WORKER_ACTIVITY_NETWORK)
1005+ PullerWorkerProtocol, install_worker_ui_factory, WORKER_ACTIVITY_NETWORK)
1006 from lp.codehosting.puller.tests import (
1007 AcceptAnythingPolicy, BlacklistPolicy, PullerWorkerMixin, WhitelistPolicy)
1008 from lp.codehosting.vfs.branchfs import (
1009 BadUrl, BadUrlLaunchpad, BadUrlScheme, BadUrlSsh, BranchPolicy,
1010- HostedBranchPolicy, ImportedBranchPolicy, MirroredBranchPolicy)
1011+ ImportedBranchPolicy, MirroredBranchPolicy)
1012 from lp.code.enums import BranchType
1013 from lp.testing import TestCase
1014 from lp.testing.factory import LaunchpadObjectFactory
1015-from lazr.uri import URI
1016
1017
1018 def get_netstrings(line):
1019@@ -89,13 +87,6 @@
1020 self.assertIsInstance(policy, MirroredBranchPolicy)
1021 self.assertIs(None, policy.stacked_on_url)
1022
1023- def testHostedOpener(self):
1024- # A PullerWorker for a hosted branch gets a HostedBranchPolicy as
1025- # the policy of its branch_mirrorer.
1026- worker = self.makePullerWorker(branch_type=BranchType.HOSTED)
1027- self.assertIsInstance(
1028- worker.branch_mirrorer.policy, HostedBranchPolicy)
1029-
1030 def testImportedOpener(self):
1031 # A PullerWorker for an imported branch gets a ImportedBranchPolicy as
1032 # the policy of its branch_mirrorer.
1033@@ -211,44 +202,20 @@
1034 source_branch.repository._format,
1035 mirrored_branch.repository._format)
1036
1037- def testRaisesStackedOnBranchNotFoundInitialMirror(self):
1038- # If the stacked-on branch cannot be found in the mirrored area on an
1039- # initial mirror, then raise StackedOnBranchNotFound. This will ensure
1040- # the puller will mirror the stacked branch as soon as the stacked-on
1041- # branch has been mirrored.
1042- self.make_branch('stacked-on-branch', format='1.6')
1043- stacked_branch = self.make_branch('source-branch', format='1.6')
1044- stacked_branch.set_stacked_on_url('../stacked-on-branch')
1045- # Make a sub-directory so that the relative URL cannot be found.
1046- self.get_transport('mirrored-area').ensure_base()
1047- # Make an empty directory with the same name as the stacked-on branch
1048- # to show that we are checking for more than just directory existence.
1049- # See bug 270757.
1050- self.get_transport('mirrored-area/stacked-on-branch').ensure_base()
1051- to_mirror = self.makePullerWorker(
1052- stacked_branch.base, self.get_url('mirrored-area/destdir'))
1053- self.assertRaises(
1054- StackedOnBranchNotFound, to_mirror.mirrorWithoutChecks)
1055-
1056- def testRaisesStackedOnBranchNotFoundRemirror(self):
1057- # If the stacked-on branch cannot be found in the mirrored area on an
1058- # update, then raise StackedOnBranchNotFound. This will ensure the
1059- # puller will mirror the stacked branch as soon as the stacked-on
1060- # branch has been mirrored.
1061- stacked_branch = self.make_branch('source-branch', format='1.6')
1062- # Make a sub-directory so that the relative URL cannot be found.
1063- self.get_transport('mirrored-area').ensure_base()
1064- # Make an empty directory with the same name as the stacked-on branch
1065- # to show that we are checking for more than just directory existence.
1066- # See bug 270757.
1067- self.get_transport('mirrored-area/stacked-on-branch').ensure_base()
1068- to_mirror = self.makePullerWorker(
1069- stacked_branch.base, self.get_url('mirrored-area/destdir'))
1070+ def testSendsStackedInfo(self):
1071+ # When the puller worker stacks a branch, it reports the stacked on
1072+ # URL to the master.
1073+ base_branch = self.make_branch('base_branch', format='1.9')
1074+ stacked_branch = self.make_branch('stacked-branch', format='1.9')
1075+ protocol_output = StringIO()
1076+ to_mirror = self.makePullerWorker(
1077+ stacked_branch.base, self.get_url('destdir'),
1078+ protocol=PullerWorkerProtocol(protocol_output),
1079+ policy=PrearrangedStackedBranchPolicy(base_branch.base))
1080 to_mirror.mirrorWithoutChecks()
1081- self.make_branch('stacked-on-branch', format='1.6')
1082- stacked_branch.set_stacked_on_url('../stacked-on-branch')
1083- self.assertRaises(
1084- StackedOnBranchNotFound, to_mirror.mirrorWithoutChecks)
1085+ self.assertEqual(
1086+ ['setStackedOn', str(to_mirror.branch_id), base_branch.base],
1087+ get_netstrings(protocol_output.getvalue()))
1088
1089 def testDoesntSendStackedInfoUnstackableFormat(self):
1090 # Mirroring an unstackable branch sends '' as the stacked-on location
1091@@ -277,40 +244,6 @@
1092 ['setStackedOn', str(to_mirror.branch_id), ''],
1093 get_netstrings(protocol_output.getvalue()))
1094
1095- def testSendsStackedInfo(self):
1096- # Mirroring a stacked branch sends the stacked-on location to the
1097- # master.
1098- base_branch = self.make_branch('base_branch', format='1.9')
1099- stacked_branch = self.make_branch(
1100- 'stacked-branch', format='1.9')
1101- stacked_branch.set_stacked_on_url(base_branch.base)
1102- protocol_output = StringIO()
1103- to_mirror = self.makePullerWorker(
1104- stacked_branch.base, self.get_url('destdir'),
1105- protocol=PullerWorkerProtocol(protocol_output))
1106- to_mirror.mirrorWithoutChecks()
1107- self.assertEqual(
1108- ['setStackedOn', str(to_mirror.branch_id),
1109- URI(stacked_branch.get_stacked_on_url()).path],
1110- get_netstrings(protocol_output.getvalue()))
1111-
1112- def testSendsStackedInfoBasedOnDestinationURL(self):
1113- # The stacked-on location sent to the master is the stacked-on
1114- # location of the _destination_ branch not the source branch in the
1115- # case that they are different.
1116- base_branch = self.make_branch('base_branch', format='1.9')
1117- stacked_branch = self.make_branch(
1118- 'stacked-branch', format='1.9')
1119- protocol_output = StringIO()
1120- to_mirror = self.makePullerWorker(
1121- stacked_branch.base, self.get_url('destdir'),
1122- protocol=PullerWorkerProtocol(protocol_output),
1123- policy=PrearrangedStackedBranchPolicy(base_branch.base))
1124- to_mirror.mirrorWithoutChecks()
1125- self.assertEqual(
1126- ['setStackedOn', str(to_mirror.branch_id), base_branch.base],
1127- get_netstrings(protocol_output.getvalue()))
1128-
1129
1130 class TestBranchMirrorerCheckAndFollowBranchReference(TestCase):
1131 """Unit tests for `BranchMirrorer.checkAndFollowBranchReference`."""
1132@@ -706,11 +639,6 @@
1133 self.protocol.setStackedOn('/~foo/bar/baz')
1134 self.assertSentNetstrings(['setStackedOn', '1', '/~foo/bar/baz'])
1135
1136- def test_mirrorDeferred(self):
1137- # Calling 'mirrorDeferred' sends 'mirrorDeferred' as a netstring.
1138- self.protocol.mirrorDeferred()
1139- self.assertSentNetstrings(['mirrorDeferred', '0'])
1140-
1141 def test_log(self):
1142 # Calling 'log' sends 'log' as a netstring and its arguments, after
1143 # formatting as a string.
1144
1145=== modified file 'lib/lp/codehosting/puller/tests/test_worker_formats.py'
1146--- lib/lp/codehosting/puller/tests/test_worker_formats.py 2010-01-21 22:54:23 +0000
1147+++ lib/lp/codehosting/puller/tests/test_worker_formats.py 2010-04-27 02:07:14 +0000
1148@@ -77,53 +77,6 @@
1149 stacked_branch_url, revision_id, stacked=True)
1150 return stacked_bzrdir.open_branch()
1151
1152- def test_stackedBranch(self):
1153- # When we mirror a stacked branch for the first time, the mirrored
1154- # branch has the same stacked-on branch.
1155- base_branch = self._createSourceBranch(
1156- RepositoryFormatKnitPack5(),
1157- BzrDirMetaFormat1(),
1158- branch_format=BzrBranchFormat7())
1159- stacked_branch = self._makeStackedBranch(
1160- 'stacked-branch', base_branch)
1161- worker = self.makePullerWorker(
1162- stacked_branch.base, self.get_url('dest'))
1163- worker.mirror()
1164- mirrored_branch = Branch.open(worker.dest)
1165- self.assertMirrored(stacked_branch, mirrored_branch)
1166- orig = stacked_branch.get_stacked_on_url()
1167- mirrored = mirrored_branch.get_stacked_on_url()
1168- self.assertEqual(URI(orig).path, mirrored)
1169-
1170- def test_reStackedBranch(self):
1171- # When we re-mirror a stacked branch we propagate any change of
1172- # stacking information.
1173- # Create and mirror a stacked branch as in the previous test.
1174- base_branch = self._createSourceBranch(
1175- RepositoryFormatKnitPack5(),
1176- BzrDirMetaFormat1(),
1177- branch_format=BzrBranchFormat7())
1178- stacked_branch = self._makeStackedBranch(
1179- 'stacked-branch', base_branch)
1180- worker = self.makePullerWorker(
1181- stacked_branch.base, self.get_url('dest'))
1182- worker.mirror()
1183-
1184- # Change the stacked-on URL and re-mirror.
1185- new_base = base_branch.bzrdir.clone(
1186- self.get_url('new-base')).open_branch()
1187- stacked_branch.set_stacked_on_url(new_base.base)
1188- worker = self.makePullerWorker(
1189- stacked_branch.base, self.get_url('dest'))
1190- worker.mirror()
1191-
1192- # Check that the mirrored branch's stacked_on_url has changed.
1193- mirrored_branch = Branch.open(worker.dest)
1194- self.assertMirrored(stacked_branch, mirrored_branch)
1195- orig = stacked_branch.get_stacked_on_url()
1196- mirrored = mirrored_branch.get_stacked_on_url()
1197- self.assertEqual(URI(orig).path, mirrored)
1198-
1199 def test_loomBranch(self):
1200 # When we mirror a loom branch for the first time, the mirrored loom
1201 # branch matches the original.
1202
1203=== modified file 'lib/lp/codehosting/puller/worker.py'
1204--- lib/lp/codehosting/puller/worker.py 2010-04-15 02:06:44 +0000
1205+++ lib/lp/codehosting/puller/worker.py 2010-04-27 02:07:14 +0000
1206@@ -33,7 +33,6 @@
1207 'install_worker_ui_factory',
1208 'PullerWorker',
1209 'PullerWorkerProtocol',
1210- 'StackedOnBranchNotFound',
1211 ]
1212
1213
1214@@ -52,10 +51,6 @@
1215 """
1216
1217
1218-class StackedOnBranchNotFound(Exception):
1219- """Couldn't find the stacked-on branch."""
1220-
1221-
1222 def get_canonical_url_for_branch_name(unique_name):
1223 """Custom implementation of canonical_url(branch) for error reporting.
1224
1225@@ -95,11 +90,6 @@
1226 def startMirroring(self):
1227 self.sendEvent('startMirroring')
1228
1229- def mirrorDeferred(self):
1230- # Called when we want to try mirroring again later without indicating
1231- # success or failure.
1232- self.sendEvent('mirrorDeferred')
1233-
1234 def mirrorSucceeded(self, revid_before, revid_after):
1235 self.sendEvent('mirrorSucceeded', revid_before, revid_after)
1236
1237@@ -270,8 +260,6 @@
1238 errors.UnstackableBranchFormat,
1239 errors.IncompatibleRepositories):
1240 stacked_on_url = None
1241- except errors.NotBranchError:
1242- raise StackedOnBranchNotFound()
1243 if stacked_on_url is None:
1244 # We use stacked_on_url == '' to mean "no stacked on location"
1245 # because XML-RPC doesn't support None.
1246@@ -283,12 +271,12 @@
1247 def mirror(self, source_branch, destination_url):
1248 """Mirror 'source_branch' to 'destination_url'."""
1249 branch = self.openDestinationBranch(source_branch, destination_url)
1250+ revid_before = branch.last_revision()
1251 # If the branch is locked, try to break it. Our special UI factory
1252 # will allow the breaking of locks that look like they were left
1253 # over from previous puller worker runs. We will block on other
1254 # locks and fail if they are not broken before the timeout expires
1255 # (currently 5 minutes).
1256- revid_before = branch.last_revision()
1257 if branch.get_physical_lock_status():
1258 branch.break_lock()
1259 self.updateBranch(source_branch, branch)
1260@@ -384,9 +372,9 @@
1261 *before* the mirroring process ran.
1262 """
1263 # Avoid circular import
1264- from lp.codehosting.vfs import get_puller_server
1265+ from lp.codehosting.vfs import get_rw_server
1266
1267- server = get_puller_server()
1268+ server = get_rw_server()
1269 server.start_server()
1270 try:
1271 source_branch = self.branch_mirrorer.open(self.source)
1272@@ -463,9 +451,6 @@
1273 except InvalidURIError, e:
1274 self._mirrorFailed(e)
1275
1276- except StackedOnBranchNotFound:
1277- self.protocol.mirrorDeferred()
1278-
1279 except (KeyboardInterrupt, SystemExit):
1280 # Do not record OOPS for those exceptions.
1281 raise
1282
1283=== modified file 'lib/lp/codehosting/vfs/__init__.py'
1284--- lib/lp/codehosting/vfs/__init__.py 2009-07-17 18:46:25 +0000
1285+++ lib/lp/codehosting/vfs/__init__.py 2010-04-27 02:07:14 +0000
1286@@ -10,7 +10,7 @@
1287 'BranchFileSystemClient',
1288 'get_lp_server',
1289 'get_multi_server',
1290- 'get_puller_server',
1291+ 'get_rw_server',
1292 'get_scanner_server',
1293 'LaunchpadServer',
1294 'make_branch_mirrorer',
1295@@ -18,7 +18,7 @@
1296
1297 from lp.codehosting.vfs.branchfs import (
1298 AsyncLaunchpadTransport, branch_id_to_path, get_lp_server,
1299- get_multi_server, get_puller_server, get_scanner_server, LaunchpadServer,
1300+ get_multi_server, get_rw_server, get_scanner_server, LaunchpadServer,
1301 make_branch_mirrorer)
1302 from lp.codehosting.vfs.branchfsclient import (
1303 BlockingProxy,BranchFileSystemClient)
1304
1305=== modified file 'lib/lp/codehosting/vfs/branchfs.py'
1306--- lib/lp/codehosting/vfs/branchfs.py 2010-04-27 02:06:51 +0000
1307+++ lib/lp/codehosting/vfs/branchfs.py 2010-04-27 02:07:14 +0000
1308@@ -55,7 +55,7 @@
1309 'DirectDatabaseLaunchpadServer',
1310 'get_lp_server',
1311 'get_multi_server',
1312- 'get_puller_server',
1313+ 'get_rw_server',
1314 'get_scanner_server',
1315 'make_branch_mirrorer',
1316 'LaunchpadInternalServer',
1317@@ -177,14 +177,18 @@
1318 'lp-mirrored:///', branchfs_endpoint, branch_transport)
1319
1320
1321-def get_puller_server():
1322- """Get a server for the Launchpad branch puller.
1323+def get_rw_server():
1324+ """Get a server that can write to the Launchpad branch vfs.
1325
1326- The server wraps up two `LaunchpadInternalServer`s. One of them points to
1327- the hosted branch area and is read-only, the other points to the mirrored
1328- area and is read/write.
1329+ You can only call this usefully on the codehost -- the transport this
1330+ server provides are backed onto file:/// URLs.
1331 """
1332- return get_multi_server(write_mirrored=True)
1333+ hosted_transport = get_chrooted_transport(
1334+ config.codehosting.mirrored_branches_root, mkdir=True)
1335+ proxy = xmlrpclib.ServerProxy(config.codehosting.branchfs_endpoint)
1336+ branchfs_endpoint = BlockingProxy(proxy)
1337+ return LaunchpadInternalServer(
1338+ 'lp-internal:///', branchfs_endpoint, hosted_transport)
1339
1340
1341 def get_multi_server(write_hosted=False, write_mirrored=False,
1342@@ -206,6 +210,8 @@
1343 directly to the database. If False, the default, use a server
1344 implementation that talks to the internal XML-RPC server.
1345 """
1346+ # XXX 2010-04-20, MichaelHudson: this function will disappear in a later
1347+ # pipe.
1348 hosted_transport = get_chrooted_transport(
1349 config.codehosting.mirrored_branches_root, mkdir=True)
1350 if not write_hosted:
1351@@ -755,23 +761,10 @@
1352 :param destination_url: The place to make the destination branch. This
1353 URL must point to a writable location.
1354 :return: The destination branch.
1355- :raises StackedOnBranchNotFound: if the branch that the destination
1356- branch will be stacked on does not yet exist in the mirrored area.
1357 """
1358 dest_transport = get_transport(destination_url)
1359 if dest_transport.has('.'):
1360 dest_transport.delete_tree('.')
1361- stacked_on_url = (
1362- self.getStackedOnURLForDestinationBranch(
1363- source_branch, destination_url))
1364- if stacked_on_url is not None:
1365- stacked_on_url = urlutils.join(destination_url, stacked_on_url)
1366- try:
1367- Branch.open(stacked_on_url)
1368- except NotBranchError:
1369- from lp.codehosting.puller.worker import (
1370- StackedOnBranchNotFound)
1371- raise StackedOnBranchNotFound()
1372 if isinstance(source_branch, LoomSupport):
1373 # Looms suck.
1374 revision_id = None
1375@@ -783,27 +776,12 @@
1376
1377 def getStackedOnURLForDestinationBranch(self, source_branch,
1378 destination_url):
1379- """Return the URL of the branch to stack the mirrored copy on.
1380-
1381- By default, we stacked the copy on the same URL as the source,
1382- relative to the new URL.
1383-
1384- :param source_branch: The branch to be mirrored.
1385- :param destination_url: The place to mirror it to.
1386- :return: The URL of the branch to stack the mirrored copy on. None if
1387- the mirrored copy should not be stacked.
1388+ """Get the stacked on URL for `source_branch`.
1389+
1390+ In particular, the URL it should be stacked on when it is mirrored to
1391+ `destination_url`.
1392 """
1393- stacked_on_url = get_stacked_on_url(source_branch)
1394- if stacked_on_url is None:
1395- return None
1396- elif '://' in stacked_on_url:
1397- # If we've gotten this far, stacked_on_url is "safe" (i.e. it's a
1398- # Launchpad URL of some form or other), so we can set the stack on
1399- # url of the destination branch to be the most access-method
1400- # compatible '/~user/project/branch' string.
1401- return URI(stacked_on_url).path
1402- else:
1403- return stacked_on_url
1404+ return None
1405
1406 def shouldFollowReferences(self):
1407 """Whether we traverse references when mirroring.
1408@@ -841,91 +819,6 @@
1409 raise NotImplementedError(self.checkOneURL)
1410
1411
1412-class HostedBranchPolicy(BranchPolicy):
1413- """Mirroring policy for HOSTED branches.
1414-
1415- In summary:
1416-
1417- - don't follow references,
1418- - assert we're pulling from a lp-hosted:/// URL.
1419- """
1420-
1421- def _bzrdirExists(self, url):
1422- """Return whether a BzrDir exists at `url`."""
1423- try:
1424- BzrDir.open(url)
1425- except NotBranchError:
1426- return False
1427- else:
1428- return True
1429-
1430- def _adjustPathURL(self, path):
1431- """Given a branch unique name, return the best stacking URL for it.
1432-
1433- If the path represents a hosted branch, then we should return a
1434- lp-hosted:/// URL. If it's mirrored, we should return a
1435- lp-mirrored:/// URL. We tell the difference by trying to open BzrDirs
1436- at the two locations -- only going as far as BzrDir to avoid getting
1437- into the mess of branch references and stacked branches.
1438- """
1439- # Avoid circular import
1440- from lp.codehosting.puller.worker import StackedOnBranchNotFound
1441- hosted_url = 'lp-hosted://' + path
1442- if self._bzrdirExists(hosted_url):
1443- return hosted_url
1444- mirrored_url = 'lp-mirrored://' + path
1445- if self._bzrdirExists(mirrored_url):
1446- return mirrored_url
1447- raise StackedOnBranchNotFound()
1448-
1449- def transformFallbackLocation(self, branch, url):
1450- """See `BranchPolicy.transformFallbackLocation`.
1451-
1452- For hosted branches, the situation is complicated.
1453-
1454- If the user pushes and the default stacking policy does it's think,
1455- the stacked_on_url will be of the form /~user/product/trunk. If this
1456- URL corresponds to a hosted branch, then we want to stack on
1457- lp-hosted:///~user/product/trunk, (although the usual URL joining
1458- rules would also do the right thing). If, however, the default stack
1459- on branch is mirrored, we need to stack on
1460- lp-mirrored:///~user/product/trunk.
1461-
1462- If the user pushes with a command line like::
1463-
1464- $ bzr push lp:~user/project/branch --stacked-on \
1465- lp:~user/project/stack-on
1466-
1467- Then the stacked_on_url will be a full bzr+ssh or http URL. We treat
1468- such URLs as if they were just the '/~user/project/branch' part, and
1469- process this as above.
1470-
1471- All other URLs are forbidden.
1472- """
1473- if '://' not in url:
1474- return self._adjustPathURL(url), False
1475- uri = URI(url)
1476- if uri.scheme not in ['http', 'bzr+ssh', 'sftp']:
1477- raise BadUrlScheme(uri.scheme, uri)
1478- launchpad_domain = config.vhost.mainsite.hostname
1479- if uri.underDomain(launchpad_domain):
1480- return self._adjustPathURL(uri.path), False
1481- else:
1482- raise BadUrl(uri)
1483-
1484- def checkOneURL(self, url):
1485- """See `BranchPolicy.checkOneURL`.
1486-
1487- If the URL we are mirroring from is anything but a
1488- lp-hosted:///~user/project/branch URL, something has gone badly wrong,
1489- so we raise AssertionError if that's happened.
1490- """
1491- uri = URI(url)
1492- if uri.scheme != 'lp-hosted':
1493- raise AssertionError(
1494- "Non-hosted url %r for hosted branch." % url)
1495-
1496-
1497 class MirroredBranchPolicy(BranchPolicy):
1498 """Mirroring policy for MIRRORED branches.
1499
1500@@ -1061,9 +954,7 @@
1501 # Avoid circular import
1502 from lp.codehosting.puller.worker import BranchMirrorer
1503
1504- if branch_type == BranchType.HOSTED:
1505- policy = HostedBranchPolicy()
1506- elif branch_type == BranchType.MIRRORED:
1507+ if branch_type == BranchType.MIRRORED:
1508 policy = MirroredBranchPolicy(mirror_stacked_on_url)
1509 elif branch_type == BranchType.IMPORTED:
1510 policy = ImportedBranchPolicy()
1511
1512=== modified file 'scripts/mirror-branch.py'
1513--- scripts/mirror-branch.py 2010-02-16 15:25:52 +0000
1514+++ scripts/mirror-branch.py 2010-04-27 02:07:14 +0000
1515@@ -45,7 +45,6 @@
1516
1517
1518 branch_type_map = {
1519- BranchType.HOSTED: 'upload',
1520 BranchType.MIRRORED: 'mirror',
1521 BranchType.IMPORTED: 'import'
1522 }