Launchpad itself

Merge lp:~mwhudson/launchpad/in-memory-launchpad-server into lp:launchpad

  1. in-memory-launchpad-server
  2. Merge into devel
Proposed by Michael Hudson-Doyle
Status: Merged
Approved by: Tim Penhey
Approved revision: no longer in the source branch.
Merged at revision: not available
Proposed branch: lp:~mwhudson/launchpad/in-memory-launchpad-server
Merge into: lp:launchpad
Diff against target: 667 lines
14 files modified
lib/lp/code/configure.zcml (+3/-0)
lib/lp/code/interfaces/branchlookup.py (+11/-0)
lib/lp/code/model/branchlookup.py (+22/-8)
lib/lp/code/model/tests/test_branchlookup.py (+43/-0)
lib/lp/code/xmlrpc/codehosting.py (+1/-19)
lib/lp/code/xmlrpc/tests/test_codehosting.py (+3/-22)
lib/lp/codehosting/rewrite.py (+14/-19)
lib/lp/codehosting/tests/test_branchdistro.py (+3/-3)
lib/lp/codehosting/vfs/branchfs.py (+60/-9)
lib/lp/codehosting/vfs/tests/test_branchfs.py (+36/-18)
lib/lp/services/tests/test_utils.py (+32/-0)
lib/lp/services/utils.py (+29/-0)
lib/lp/testing/__init__.py (+4/-2)
scripts/branch-distro.py (+2/-1)
To merge this branch: bzr merge lp:~mwhudson/launchpad/in-memory-launchpad-server
Reviewer Review Type Date Requested Status
Tim Penhey (community) Approve
Review via email: mp+13613@code.launchpad.net

Commit message

Add a way to access branches by querying the database directly rather than via an XML-RPC call, and use it in the branch-distro.py script.

To post a comment you must log in.
Revision history for this message
Michael Hudson-Doyle (mwhudson) wrote :

Hi,

This branch adds a way of translating branch paths from external to internal by direct communication with the database, and uses it for the branch-distro.py script. Tests on staging became about 50x faster as a result of this change.

Cheers,
mwh

Revision history for this message
Tim Penhey (thumper) wrote :

I think this is actually pretty good.

Some of the tests are missing comments/docstrings, but the combination of
class name and method name makes it pretty clear what is being tested.

  merge approved

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/code/configure.zcml'
--- lib/lp/code/configure.zcml 2009-10-15 05:45:57 +0000
+++ lib/lp/code/configure.zcml 2009-10-20 21:56:15 +0000
@@ -355,6 +355,9 @@
355 provides="lp.code.interfaces.branchlookup.ILinkedBranchTraverser">355 provides="lp.code.interfaces.branchlookup.ILinkedBranchTraverser">
356 <allow interface="lp.code.interfaces.branchlookup.ILinkedBranchTraverser"/>356 <allow interface="lp.code.interfaces.branchlookup.ILinkedBranchTraverser"/>
357 </securedutility>357 </securedutility>
358 <adapter factory="lp.code.model.branchlookup.ProductTraversable" />
359 <adapter factory="lp.code.model.branchlookup.DistributionTraversable" />
360 <adapter factory="lp.code.model.branchlookup.DistroSeriesTraversable" />
358 <securedutility361 <securedutility
359 class="lp.code.model.branchscanner.BranchScanner"362 class="lp.code.model.branchscanner.BranchScanner"
360 provides="lp.code.interfaces.branchscanner.IBranchScanner">363 provides="lp.code.interfaces.branchscanner.IBranchScanner">
361364
=== modified file 'lib/lp/code/interfaces/branchlookup.py'
--- lib/lp/code/interfaces/branchlookup.py 2009-07-19 23:17:34 +0000
+++ lib/lp/code/interfaces/branchlookup.py 2009-10-20 21:56:15 +0000
@@ -68,6 +68,17 @@
68 Return None if no match was found.68 Return None if no match was found.
69 """69 """
7070
71 def getIdAndTrailingPath(self, path, from_slave=False):
72 """Return id of and path within the branch identified by the `path`.
73
74 To explain by example, if the branch with id 5 has unique name
75 '~user/project/name', getIdAndTrailingPath('~user/project/name/foo')
76 will return (5, '/foo').
77
78 :return: ``(branch_id, trailing_path)``, both will be ``None`` if no
79 branch is identified.
80 """
81
71 def uriToUniqueName(uri):82 def uriToUniqueName(uri):
72 """Return the unique name for the URI, if the URI is on codehosting.83 """Return the unique name for the URI, if the URI is on codehosting.
7384
7485
=== modified file 'lib/lp/code/model/branchlookup.py'
--- lib/lp/code/model/branchlookup.py 2009-07-19 23:17:34 +0000
+++ lib/lp/code/model/branchlookup.py 2009-10-20 21:56:15 +0000
@@ -8,8 +8,7 @@
8# then get the IBranchLookup utility.8# then get the IBranchLookup utility.
9__all__ = []9__all__ = []
1010
11from zope.component import (11from zope.component import adapts, getUtility, queryMultiAdapter
12 adapts, getSiteManager, getUtility, queryMultiAdapter)
13from zope.interface import implements12from zope.interface import implements
1413
15from storm.expr import Join14from storm.expr import Join
@@ -37,7 +36,9 @@
37from lp.registry.interfaces.productseries import NoSuchProductSeries36from lp.registry.interfaces.productseries import NoSuchProductSeries
38from lp.registry.interfaces.sourcepackagename import (37from lp.registry.interfaces.sourcepackagename import (
39 NoSuchSourcePackageName)38 NoSuchSourcePackageName)
39from lp.services.utils import iter_split
40from canonical.launchpad.validators.name import valid_name40from canonical.launchpad.validators.name import valid_name
41from canonical.launchpad.interfaces.lpstorm import IMasterStore, ISlaveStore
41from canonical.launchpad.webapp.authorization import check_permission42from canonical.launchpad.webapp.authorization import check_permission
42from canonical.launchpad.webapp.interfaces import (43from canonical.launchpad.webapp.interfaces import (
43 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)44 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
@@ -160,12 +161,6 @@
160 return sourcepackage.getSuiteSourcePackage(self.pocket)161 return sourcepackage.getSuiteSourcePackage(self.pocket)
161162
162163
163sm = getSiteManager()
164sm.registerAdapter(ProductTraversable)
165sm.registerAdapter(DistributionTraversable)
166sm.registerAdapter(DistroSeriesTraversable)
167
168
169class LinkedBranchTraverser:164class LinkedBranchTraverser:
170 """Utility for traversing to objects that can have linked branches."""165 """Utility for traversing to objects that can have linked branches."""
171166
@@ -259,6 +254,25 @@
259 return None254 return None
260 return self._getBranchInNamespace(namespace_data, branch_name)255 return self._getBranchInNamespace(namespace_data, branch_name)
261256
257 def getIdAndTrailingPath(self, path, from_slave=False):
258 """See `IBranchLookup`. """
259 if from_slave:
260 store = ISlaveStore(Branch)
261 else:
262 store = IMasterStore(Branch)
263 prefixes = []
264 for first, second in iter_split(path[1:], '/'):
265 prefixes.append(first)
266 result = store.find(
267 (Branch.id, Branch.unique_name),
268 Branch.unique_name.is_in(prefixes), Branch.private == False).one()
269 if result is None:
270 return None, None
271 else:
272 branch_id, unique_name = result
273 trailing = path[len(unique_name) + 1:]
274 return branch_id, trailing
275
262 def _getBranchInNamespace(self, namespace_data, branch_name):276 def _getBranchInNamespace(self, namespace_data, branch_name):
263 if namespace_data['product'] == '+junk':277 if namespace_data['product'] == '+junk':
264 return self._getPersonalBranch(278 return self._getPersonalBranch(
265279
=== modified file 'lib/lp/code/model/tests/test_branchlookup.py'
--- lib/lp/code/model/tests/test_branchlookup.py 2009-08-28 06:39:38 +0000
+++ lib/lp/code/model/tests/test_branchlookup.py 2009-10-20 21:56:15 +0000
@@ -63,6 +63,49 @@
63 self.assertEqual(branch, found_branch)63 self.assertEqual(branch, found_branch)
6464
6565
66class TestGetIdAndTrailingPath(TestCaseWithFactory):
67 """Tests for `IBranchLookup.getIdAndTrailingPath`."""
68
69 layer = DatabaseFunctionalLayer
70
71 def setUp(self):
72 TestCaseWithFactory.setUp(self)
73 self.branch_set = getUtility(IBranchLookup)
74
75 def test_not_found(self):
76 unused_name = self.factory.getUniqueString()
77 result = self.branch_set.getIdAndTrailingPath('/' + unused_name)
78 self.assertEqual((None, None), result)
79
80 def test_junk(self):
81 branch = self.factory.makePersonalBranch()
82 result = self.branch_set.getIdAndTrailingPath('/' + branch.unique_name)
83 self.assertEqual((branch.id, ''), result)
84
85 def test_product(self):
86 branch = self.factory.makeProductBranch()
87 result = self.branch_set.getIdAndTrailingPath('/' + branch.unique_name)
88 self.assertEqual((branch.id, ''), result)
89
90 def test_source_package(self):
91 branch = self.factory.makePackageBranch()
92 result = self.branch_set.getIdAndTrailingPath('/' + branch.unique_name)
93 self.assertEqual((branch.id, ''), result)
94
95 def test_trailing_slash(self):
96 branch = self.factory.makeAnyBranch()
97 result = self.branch_set.getIdAndTrailingPath(
98 '/' + branch.unique_name + '/')
99 self.assertEqual((branch.id, '/'), result)
100
101 def test_trailing_path(self):
102 branch = self.factory.makeAnyBranch()
103 path = self.factory.getUniqueString()
104 result = self.branch_set.getIdAndTrailingPath(
105 '/' + branch.unique_name + '/' + path)
106 self.assertEqual((branch.id, '/' + path), result)
107
108
66class TestGetByPath(TestCaseWithFactory):109class TestGetByPath(TestCaseWithFactory):
67 """Test `IBranchLookup.getByLPPath`."""110 """Test `IBranchLookup.getByLPPath`."""
68111
69112
=== modified file 'lib/lp/code/xmlrpc/codehosting.py'
--- lib/lp/code/xmlrpc/codehosting.py 2009-07-23 02:07:29 +0000
+++ lib/lp/code/xmlrpc/codehosting.py 2009-10-20 21:56:15 +0000
@@ -8,7 +8,6 @@
8 'BranchFileSystem',8 'BranchFileSystem',
9 'BranchPuller',9 'BranchPuller',
10 'datetime_from_tuple',10 'datetime_from_tuple',
11 'iter_split',
12 ]11 ]
1312
1413
@@ -37,6 +36,7 @@
37from lp.registry.interfaces.person import IPersonSet, NoSuchPerson36from lp.registry.interfaces.person import IPersonSet, NoSuchPerson
38from lp.registry.interfaces.product import NoSuchProduct37from lp.registry.interfaces.product import NoSuchProduct
39from lp.services.scripts.interfaces.scriptactivity import IScriptActivitySet38from lp.services.scripts.interfaces.scriptactivity import IScriptActivitySet
39from lp.services.utils import iter_split
40from canonical.launchpad.validators import LaunchpadValidationError40from canonical.launchpad.validators import LaunchpadValidationError
41from canonical.launchpad.webapp import LaunchpadXMLRPCView41from canonical.launchpad.webapp import LaunchpadXMLRPCView
42from canonical.launchpad.webapp.authorization import check_permission42from canonical.launchpad.webapp.authorization import check_permission
@@ -337,21 +337,3 @@
337 raise faults.PathTranslationError(path)337 raise faults.PathTranslationError(path)
338 return run_with_login(requester_id, translate_path)338 return run_with_login(requester_id, translate_path)
339339
340
341def iter_split(string, splitter):
342 """Iterate over ways to split 'string' in two with 'splitter'.
343
344 If 'string' is empty, then yield nothing. Otherwise, yield tuples like
345 ('a/b/c', ''), ('a/b', 'c'), ('a', 'b/c') for a string 'a/b/c' and a
346 splitter '/'.
347
348 The tuples are yielded such that the first tuple has everything in the
349 first tuple. With each iteration, the first element gets smaller and the
350 second gets larger. It stops iterating just before it would have to yield
351 ('', 'a/b/c').
352 """
353 if string == '':
354 return
355 tokens = string.split(splitter)
356 for i in reversed(range(1, len(tokens) + 1)):
357 yield splitter.join(tokens[:i]), splitter.join(tokens[i:])
358340
=== modified file 'lib/lp/code/xmlrpc/tests/test_codehosting.py'
--- lib/lp/code/xmlrpc/tests/test_codehosting.py 2009-07-17 00:26:05 +0000
+++ lib/lp/code/xmlrpc/tests/test_codehosting.py 2009-10-20 21:56:15 +0000
@@ -23,7 +23,7 @@
23from lp.code.interfaces.codehosting import (23from lp.code.interfaces.codehosting import (
24 BRANCH_TRANSPORT, CONTROL_TRANSPORT)24 BRANCH_TRANSPORT, CONTROL_TRANSPORT)
25from canonical.launchpad.interfaces.launchpad import ILaunchBag25from canonical.launchpad.interfaces.launchpad import ILaunchBag
26from lp.testing import TestCase, TestCaseWithFactory26from lp.testing import TestCaseWithFactory
27from lp.testing.factory import LaunchpadObjectFactory27from lp.testing.factory import LaunchpadObjectFactory
28from canonical.launchpad.webapp.interfaces import NotFoundError28from canonical.launchpad.webapp.interfaces import NotFoundError
29from canonical.launchpad.xmlrpc import faults29from canonical.launchpad.xmlrpc import faults
@@ -36,7 +36,7 @@
36from lp.code.model.tests.test_branchpuller import AcquireBranchToPullTests36from lp.code.model.tests.test_branchpuller import AcquireBranchToPullTests
37from lp.code.xmlrpc.codehosting import (37from lp.code.xmlrpc.codehosting import (
38 BranchFileSystem, BranchPuller, LAUNCHPAD_ANONYMOUS, LAUNCHPAD_SERVICES,38 BranchFileSystem, BranchPuller, LAUNCHPAD_ANONYMOUS, LAUNCHPAD_SERVICES,
39 iter_split, run_with_login)39 run_with_login)
4040
4141
42UTC = pytz.timezone('UTC')42UTC = pytz.timezone('UTC')
@@ -1109,23 +1109,6 @@
1109 trailing_path='.bzr')1109 trailing_path='.bzr')
11101110
11111111
1112class TestIterateSplit(TestCase):
1113 """Tests for iter_split."""
1114
1115 def test_iter_split(self):
1116 # iter_split loops over each way of splitting a string in two using
1117 # the given splitter.
1118 self.assertEqual([('one', '')], list(iter_split('one', '/')))
1119 self.assertEqual([], list(iter_split('', '/')))
1120 self.assertEqual(
1121 [('one/two', ''), ('one', 'two')],
1122 list(iter_split('one/two', '/')))
1123 self.assertEqual(
1124 [('one/two/three', ''), ('one/two', 'three'),
1125 ('one', 'two/three')],
1126 list(iter_split('one/two/three', '/')))
1127
1128
1129class LaunchpadDatabaseFrontend:1112class LaunchpadDatabaseFrontend:
1130 """A 'frontend' to Launchpad's branch services.1113 """A 'frontend' to Launchpad's branch services.
11311114
@@ -1182,7 +1165,5 @@
1182 'layer': FunctionalLayer}),1165 'layer': FunctionalLayer}),
1183 ]1166 ]
1184 multiply_tests(puller_tests, scenarios, suite)1167 multiply_tests(puller_tests, scenarios, suite)
1185 suite.addTests(1168 suite.addTests(loader.loadTestsFromTestCase(TestRunWithLogin))
1186 map(loader.loadTestsFromTestCase,
1187 [TestRunWithLogin, TestIterateSplit]))
1188 return suite1169 return suite
11891170
=== modified file 'lib/lp/codehosting/rewrite.py'
--- lib/lp/codehosting/rewrite.py 2009-07-23 23:42:47 +0000
+++ lib/lp/codehosting/rewrite.py 2009-10-20 21:56:15 +0000
@@ -8,11 +8,14 @@
88
9from bzrlib import urlutils9from bzrlib import urlutils
1010
11from canonical.launchpad.interfaces import ISlaveStore11from zope.component import getUtility
12from lp.code.model.branch import Branch12
13from canonical.config import config
14
15from lp.code.interfaces.branchlookup import IBranchLookup
13from lp.codehosting.vfs import branch_id_to_path16from lp.codehosting.vfs import branch_id_to_path
17from lp.services.utils import iter_split
1418
15from canonical.config import config
1619
17__all__ = ['BranchRewriter']20__all__ = ['BranchRewriter']
1821
@@ -31,7 +34,6 @@
31 else:34 else:
32 self._now = _now35 self._now = _now
33 self.logger = logger36 self.logger = logger
34 self.store = ISlaveStore(Branch)
35 self._cache = {}37 self._cache = {}
3638
37 def _codebrowse_url(self, path):39 def _codebrowse_url(self, path):
@@ -45,26 +47,19 @@
45 In addition this method returns whether the answer can from the cache47 In addition this method returns whether the answer can from the cache
46 or from the database.48 or from the database.
47 """49 """
48 parts = location[1:].split('/')50 for first, second in iter_split(location[1:], '/'):
49 prefixes = []51 if first in self._cache:
50 for i in range(1, len(parts) + 1):52 branch_id, inserted_time = self._cache[first]
51 prefix = '/'.join(parts[:i])
52 if prefix in self._cache:
53 branch_id, inserted_time = self._cache[prefix]
54 if (self._now() < inserted_time +53 if (self._now() < inserted_time +
55 config.codehosting.branch_rewrite_cache_lifetime):54 config.codehosting.branch_rewrite_cache_lifetime):
56 trailing = location[len(prefix) + 1:]55 return branch_id, second, "HIT"
57 return branch_id, trailing, "HIT"56 branch_id, trailing = getUtility(IBranchLookup).getIdAndTrailingPath(
58 prefixes.append(prefix)57 location, from_slave=True)
59 result = self.store.find(58 if branch_id is None:
60 (Branch.id, Branch.unique_name),
61 Branch.unique_name.is_in(prefixes), Branch.private == False).one()
62 if result is None:
63 return None, None, "MISS"59 return None, None, "MISS"
64 else:60 else:
65 branch_id, unique_name = result61 unique_name = location[1:-len(trailing)]
66 self._cache[unique_name] = (branch_id, self._now())62 self._cache[unique_name] = (branch_id, self._now())
67 trailing = location[len(unique_name) + 1:]
68 return branch_id, trailing, "MISS"63 return branch_id, trailing, "MISS"
6964
70 def rewriteLine(self, resource_location):65 def rewriteLine(self, resource_location):
7166
=== modified file 'lib/lp/codehosting/tests/test_branchdistro.py'
--- lib/lp/codehosting/tests/test_branchdistro.py 2009-10-14 00:18:23 +0000
+++ lib/lp/codehosting/tests/test_branchdistro.py 2009-10-20 21:56:15 +0000
@@ -25,7 +25,7 @@
25import transaction25import transaction
2626
27from canonical.config import config27from canonical.config import config
28from canonical.testing.layers import ZopelessAppServerLayer28from canonical.testing.layers import LaunchpadZopelessLayer
29from canonical.launchpad.scripts.logger import FakeLogger, QuietFakeLogger29from canonical.launchpad.scripts.logger import FakeLogger, QuietFakeLogger
3030
31from lp.codehosting.branchdistro import DistroBrancher, switch_branches31from lp.codehosting.branchdistro import DistroBrancher, switch_branches
@@ -104,11 +104,11 @@
104class TestDistroBrancher(TestCaseWithFactory):104class TestDistroBrancher(TestCaseWithFactory):
105 """Tests for `DistroBrancher`."""105 """Tests for `DistroBrancher`."""
106106
107 layer = ZopelessAppServerLayer107 layer = LaunchpadZopelessLayer
108108
109 def setUp(self):109 def setUp(self):
110 TestCaseWithFactory.setUp(self)110 TestCaseWithFactory.setUp(self)
111 self.useBzrBranches(real_server=True)111 self.useBzrBranches(real_server=True, direct_database=True)
112112
113 def makeOfficialPackageBranch(self, distroseries=None):113 def makeOfficialPackageBranch(self, distroseries=None):
114 """Make an official package branch with an underlying bzr branch."""114 """Make an official package branch with an underlying bzr branch."""
115115
=== modified file 'lib/lp/codehosting/vfs/branchfs.py'
--- lib/lp/codehosting/vfs/branchfs.py 2009-07-17 18:46:25 +0000
+++ lib/lp/codehosting/vfs/branchfs.py 2009-10-20 21:56:15 +0000
@@ -53,6 +53,7 @@
53 'BadUrlSsh',53 'BadUrlSsh',
54 'branch_id_to_path',54 'branch_id_to_path',
55 'BranchPolicy',55 'BranchPolicy',
56 'DirectDatabaseLaunchpadServer',
56 'get_lp_server',57 'get_lp_server',
57 'get_multi_server',58 'get_multi_server',
58 'get_puller_server',59 'get_puller_server',
@@ -77,6 +78,7 @@
77from twisted.internet import defer78from twisted.internet import defer
78from twisted.python import failure79from twisted.python import failure
7980
81from zope.component import getUtility
80from zope.interface import implements, Interface82from zope.interface import implements, Interface
8183
82from lp.codehosting.vfs.branchfsclient import (84from lp.codehosting.vfs.branchfsclient import (
@@ -87,6 +89,7 @@
87 get_chrooted_transport, get_readonly_transport, TranslationError)89 get_chrooted_transport, get_readonly_transport, TranslationError)
88from canonical.config import config90from canonical.config import config
89from lp.code.enums import BranchType91from lp.code.enums import BranchType
92from lp.code.interfaces.branchlookup import IBranchLookup
90from lp.code.interfaces.codehosting import (93from lp.code.interfaces.codehosting import (
91 BRANCH_TRANSPORT, CONTROL_TRANSPORT, LAUNCHPAD_SERVICES)94 BRANCH_TRANSPORT, CONTROL_TRANSPORT, LAUNCHPAD_SERVICES)
92from canonical.launchpad.xmlrpc import faults95from canonical.launchpad.xmlrpc import faults
@@ -180,20 +183,25 @@
180 return get_multi_server(write_mirrored=True)183 return get_multi_server(write_mirrored=True)
181184
182185
183def get_multi_server(write_hosted=False, write_mirrored=False):186def get_multi_server(write_hosted=False, write_mirrored=False,
187 direct_database=False):
184 """Get a server with access to both mirrored and hosted areas.188 """Get a server with access to both mirrored and hosted areas.
185189
186 The server wraps up two `LaunchpadInternalServer`s. One of them points to190 The server wraps up two `LaunchpadInternalServer`s or
187 the hosted branch area, the other points to the mirrored area.191 `DirectDatabaseLaunchpadServer`s. One server points to the hosted branch
192 area and the other points to the mirrored area.
188193
189 Write permision defaults to False, but can be overridden.194 Write permision defaults to False, but can be overridden.
195
190 :param write_hosted: if True, lp-hosted URLs are writeable. Otherwise,196 :param write_hosted: if True, lp-hosted URLs are writeable. Otherwise,
191 they are read-only.197 they are read-only.
192 :param write_mirrored: if True, lp-mirrored URLs are writeable.198 :param write_mirrored: if True, lp-mirrored URLs are writeable.
193 Otherwise, they are read-only.199 Otherwise, they are read-only.
200
201 :param direct_database: if True, use a server implementation that talks
202 directly to the database. If False, the default, use a server
203 implementation that talks to the internal XML-RPC server.
194 """204 """
195 proxy = xmlrpclib.ServerProxy(config.codehosting.branchfs_endpoint)
196 branchfs_endpoint = BlockingProxy(proxy)
197 hosted_transport = get_chrooted_transport(205 hosted_transport = get_chrooted_transport(
198 config.codehosting.hosted_branches_root, mkdir=True)206 config.codehosting.hosted_branches_root, mkdir=True)
199 if not write_hosted:207 if not write_hosted:
@@ -202,10 +210,16 @@
202 config.codehosting.mirrored_branches_root, mkdir=True)210 config.codehosting.mirrored_branches_root, mkdir=True)
203 if not write_mirrored:211 if not write_mirrored:
204 mirrored_transport = get_readonly_transport(mirrored_transport)212 mirrored_transport = get_readonly_transport(mirrored_transport)
205 hosted_server = LaunchpadInternalServer(213 if direct_database:
206 'lp-hosted:///', branchfs_endpoint, hosted_transport)214 make_server = DirectDatabaseLaunchpadServer
207 mirrored_server = LaunchpadInternalServer(215 else:
208 'lp-mirrored:///', branchfs_endpoint, mirrored_transport)216 proxy = xmlrpclib.ServerProxy(config.codehosting.branchfs_endpoint)
217 branchfs_endpoint = BlockingProxy(proxy)
218 def make_server(scheme, transport):
219 return LaunchpadInternalServer(
220 scheme, branchfs_endpoint, transport)
221 hosted_server = make_server('lp-hosted:///', hosted_transport)
222 mirrored_server = make_server('lp-mirrored:///', mirrored_transport)
209 return _MultiServer(hosted_server, mirrored_server)223 return _MultiServer(hosted_server, mirrored_server)
210224
211225
@@ -427,6 +441,43 @@
427 self.tearDown()441 self.tearDown()
428442
429443
444class DirectDatabaseLaunchpadServer(AsyncVirtualServer):
445 def __init__(self, scheme, branch_transport):
446 AsyncVirtualServer.__init__(self, scheme)
447 self._transport_dispatch = BranchTransportDispatch(branch_transport)
448
449 def setUp(self):
450 super(DirectDatabaseLaunchpadServer, self).setUp()
451 try:
452 self._transport_dispatch.base_transport.ensure_base()
453 except TransportNotPossible:
454 pass
455
456 def destroy(self):
457 """Delete the on-disk branches and tear down."""
458 self._transport_dispatch.base_transport.delete_tree('.')
459 self.tearDown()
460
461 def translateVirtualPath(self, virtual_url_fragment):
462 """See `AsyncVirtualServer.translateVirtualPath`.
463
464 This implementation connects to the database directly.
465 """
466 deferred = defer.succeed(
467 getUtility(IBranchLookup).getIdAndTrailingPath(
468 virtual_url_fragment))
469
470 def process_result((branch_id, trailing)):
471 if branch_id is None:
472 raise NoSuchFile(virtual_url_fragment)
473 else:
474 return self._transport_dispatch.makeTransport(
475 (BRANCH_TRANSPORT, dict(id=branch_id), trailing[1:]))
476
477 deferred.addCallback(process_result)
478 return deferred
479
480
430class AsyncLaunchpadTransport(AsyncVirtualTransport):481class AsyncLaunchpadTransport(AsyncVirtualTransport):
431 """Virtual transport to implement the Launchpad VFS for branches.482 """Virtual transport to implement the Launchpad VFS for branches.
432483
433484
=== modified file 'lib/lp/codehosting/vfs/tests/test_branchfs.py'
--- lib/lp/codehosting/vfs/tests/test_branchfs.py 2009-06-25 04:06:00 +0000
+++ lib/lp/codehosting/vfs/tests/test_branchfs.py 2009-10-20 21:56:15 +0000
@@ -24,9 +24,9 @@
24from twisted.trial.unittest import TestCase as TrialTestCase24from twisted.trial.unittest import TestCase as TrialTestCase
2525
26from lp.codehosting.vfs.branchfs import (26from lp.codehosting.vfs.branchfs import (
27 AsyncLaunchpadTransport, branch_id_to_path, LaunchpadInternalServer,27 AsyncLaunchpadTransport, BranchTransportDispatch,
28 LaunchpadServer, BranchTransportDispatch, TransportDispatch,28 DirectDatabaseLaunchpadServer, LaunchpadInternalServer, LaunchpadServer,
29 UnknownTransportType)29 TransportDispatch, UnknownTransportType, branch_id_to_path)
30from lp.codehosting.bzrutils import ensure_base30from lp.codehosting.bzrutils import ensure_base
31from lp.codehosting.inmemory import InMemoryFrontend, XMLRPCWrapper31from lp.codehosting.inmemory import InMemoryFrontend, XMLRPCWrapper
32from lp.codehosting.sftp import FatLocalTransport32from lp.codehosting.sftp import FatLocalTransport
@@ -34,8 +34,8 @@
34from lp.code.enums import BranchType34from lp.code.enums import BranchType
35from lp.code.interfaces.codehosting import (35from lp.code.interfaces.codehosting import (
36 BRANCH_TRANSPORT, CONTROL_TRANSPORT)36 BRANCH_TRANSPORT, CONTROL_TRANSPORT)
37from lp.testing import TestCase37from lp.testing import TestCase, TestCaseWithFactory
38from canonical.testing import TwistedLayer38from canonical.testing import TwistedLayer, ZopelessDatabaseLayer
3939
4040
41def branch_to_path(branch, add_slash=True):41def branch_to_path(branch, add_slash=True):
@@ -293,18 +293,8 @@
293 self.assertEqual('lp-%d:///' % id(self.server), self.server.get_url())293 self.assertEqual('lp-%d:///' % id(self.server), self.server.get_url())
294294
295295
296class TestLaunchpadInternalServer(MixinBaseLaunchpadServerTests,296class LaunchpadInternalServerTests:
297 TrialTestCase, BzrTestCase):297 """Tests for the internal server classes, used by e.g. the scanner."""
298 """Tests for the LaunchpadInternalServer, used by the puller and scanner.
299 """
300
301 def setUp(self):
302 BzrTestCase.setUp(self)
303 MixinBaseLaunchpadServerTests.setUp(self)
304
305 def getLaunchpadServer(self, authserver, user_id):
306 return LaunchpadInternalServer(
307 'lp-test:///', XMLRPCWrapper(authserver), MemoryTransport())
308298
309 def test_translate_branch_path(self):299 def test_translate_branch_path(self):
310 branch = self.factory.makeAnyBranch()300 branch = self.factory.makeAnyBranch()
@@ -314,7 +304,7 @@
314 expected_transport, expected_path = dispatch304 expected_transport, expected_path = dispatch
315305
316 deferred = self.server.translateVirtualPath(306 deferred = self.server.translateVirtualPath(
317 '%s/.bzr/README' % (branch.unique_name,))307 '/%s/.bzr/README' % (branch.unique_name,))
318 def check_branch_transport((transport, path)):308 def check_branch_transport((transport, path)):
319 self.assertEqual(expected_path, path)309 self.assertEqual(expected_path, path)
320 # Can't test for equality of transports, since URLs and object310 # Can't test for equality of transports, since URLs and object
@@ -348,6 +338,34 @@
348 BzrDir.open_containing_from_transport, transport)338 BzrDir.open_containing_from_transport, transport)
349339
350340
341class TestLaunchpadInternalServer(MixinBaseLaunchpadServerTests,
342 TrialTestCase, BzrTestCase,
343 LaunchpadInternalServerTests):
344 """Tests for `LaunchpadInternalServer`, used by the puller and scanner."""
345
346 def setUp(self):
347 BzrTestCase.setUp(self)
348 MixinBaseLaunchpadServerTests.setUp(self)
349
350 def getLaunchpadServer(self, authserver, user_id):
351 return LaunchpadInternalServer(
352 'lp-test:///', XMLRPCWrapper(authserver), MemoryTransport())
353
354
355class TestDirectDatabaseLaunchpadServer(TestCaseWithFactory, TrialTestCase,
356 LaunchpadInternalServerTests):
357 """Tests for `DirectDatabaseLaunchpadServer`."""
358
359 layer = ZopelessDatabaseLayer
360
361 def setUp(self):
362 super(TestDirectDatabaseLaunchpadServer, self).setUp()
363 self.requester = self.factory.makePerson()
364 self.server = DirectDatabaseLaunchpadServer(
365 'lp-test://', MemoryTransport())
366
367
368
351class TestAsyncVirtualTransport(TrialTestCase, TestCaseInTempDir):369class TestAsyncVirtualTransport(TrialTestCase, TestCaseInTempDir):
352 """Tests for `AsyncVirtualTransport`."""370 """Tests for `AsyncVirtualTransport`."""
353371
354372
=== added file 'lib/lp/services/tests/test_utils.py'
--- lib/lp/services/tests/test_utils.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/tests/test_utils.py 2009-10-20 21:56:15 +0000
@@ -0,0 +1,32 @@
1# Copyright 2009 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Module docstring goes here."""
5
6__metaclass__ = type
7
8import unittest
9
10from lp.services.utils import iter_split
11from lp.testing import TestCase
12
13
14class TestIterateSplit(TestCase):
15 """Tests for iter_split."""
16
17 def test_iter_split(self):
18 # iter_split loops over each way of splitting a string in two using
19 # the given splitter.
20 self.assertEqual([('one', '')], list(iter_split('one', '/')))
21 self.assertEqual([], list(iter_split('', '/')))
22 self.assertEqual(
23 [('one/two', ''), ('one', 'two')],
24 list(iter_split('one/two', '/')))
25 self.assertEqual(
26 [('one/two/three', ''), ('one/two', 'three'),
27 ('one', 'two/three')],
28 list(iter_split('one/two/three', '/')))
29
30
31def test_suite():
32 return unittest.TestLoader().loadTestsFromName(__name__)
033
=== added file 'lib/lp/services/utils.py'
--- lib/lp/services/utils.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/utils.py 2009-10-20 21:56:15 +0000
@@ -0,0 +1,29 @@
1# Copyright 2009 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4
5"""Implementations of the XML-RPC APIs for codehosting."""
6
7__metaclass__ = type
8__all__ = [
9 'iter_split',
10 ]
11
12
13def iter_split(string, splitter):
14 """Iterate over ways to split 'string' in two with 'splitter'.
15
16 If 'string' is empty, then yield nothing. Otherwise, yield tuples like
17 ('a/b/c', ''), ('a/b', 'c'), ('a', 'b/c') for a string 'a/b/c' and a
18 splitter '/'.
19
20 The tuples are yielded such that the first tuple has everything in the
21 first tuple. With each iteration, the first element gets smaller and the
22 second gets larger. It stops iterating just before it would have to yield
23 ('', 'a/b/c').
24 """
25 if string == '':
26 return
27 tokens = string.split(splitter)
28 for i in reversed(range(1, len(tokens) + 1)):
29 yield splitter.join(tokens[:i]), splitter.join(tokens[i:])
030
=== modified file 'lib/lp/testing/__init__.py'
--- lib/lp/testing/__init__.py 2009-09-16 21:22:12 +0000
+++ lib/lp/testing/__init__.py 2009-10-20 21:56:15 +0000
@@ -572,7 +572,7 @@
572 os.environ['BZR_HOME'] = os.getcwd()572 os.environ['BZR_HOME'] = os.getcwd()
573 self.addCleanup(restore_bzr_home)573 self.addCleanup(restore_bzr_home)
574574
575 def useBzrBranches(self, real_server=False):575 def useBzrBranches(self, real_server=False, direct_database=False):
576 """Prepare for using bzr branches.576 """Prepare for using bzr branches.
577577
578 This sets up support for lp-hosted and lp-mirrored URLs,578 This sets up support for lp-hosted and lp-mirrored URLs,
@@ -586,7 +586,9 @@
586 self.useTempBzrHome()586 self.useTempBzrHome()
587 self.real_bzr_server = real_server587 self.real_bzr_server = real_server
588 if real_server:588 if real_server:
589 server = get_multi_server(write_hosted=True, write_mirrored=True)589 server = get_multi_server(
590 write_hosted=True, write_mirrored=True,
591 direct_database=direct_database)
590 server.setUp()592 server.setUp()
591 self.addCleanup(server.destroy)593 self.addCleanup(server.destroy)
592 else:594 else:
593595
=== modified file 'scripts/branch-distro.py'
--- scripts/branch-distro.py 2009-10-13 23:50:17 +0000
+++ scripts/branch-distro.py 2009-10-20 21:56:15 +0000
@@ -24,7 +24,8 @@
24 if len(self.args) != 3:24 if len(self.args) != 3:
25 self.parser.error("Wrong number of arguments.")25 self.parser.error("Wrong number of arguments.")
26 brancher = DistroBrancher.fromNames(self.logger, *self.args)26 brancher = DistroBrancher.fromNames(self.logger, *self.args)
27 server = get_multi_server(write_mirrored=True, write_hosted=True)27 server = get_multi_server(
28 write_mirrored=True, write_hosted=True, direct_database=True)
28 server.setUp()29 server.setUp()
29 try:30 try:
30 if self.options.check:31 if self.options.check: