Launchpad itself

Merge lp:~mwhudson/launchpad/more-git-caches into lp:launchpad

  1. more-git-caches
  2. Merge into devel
Proposed by Michael Hudson-Doyle
Status: Merged
Approved by: Tim Penhey
Approved revision: no longer in the source branch.
Merged at revision: not available
Proposed branch: lp:~mwhudson/launchpad/more-git-caches
Merge into: lp:launchpad
Diff against target: 182 lines (+61/-20)
4 files modified
lib/canonical/config/schema-lazr.conf (+5/-3)
lib/lp/codehosting/codeimport/tests/test_worker.py (+36/-10)
lib/lp/codehosting/codeimport/worker.py (+18/-5)
utilities/sourcedeps.conf (+2/-2)
To merge this branch: bzr merge lp:~mwhudson/launchpad/more-git-caches
Reviewer Review Type Date Requested Status
Tim Penhey (community) Approve
Review via email: mp+23069@code.launchpad.net

Commit message

Update to the new, massively faster bzr-git and a few Launchpad changes to cope with the new cache structure.

Description of the change

I haven't landed the bzr-git and dulwich updates yet, so sourcedeps.conf needs to change but otherwise this branch adapts us to the new bzr-git hotness.

To post a comment you must log in.
Revision history for this message
Tim Penhey (thumper) wrote :

You have git_revisions_import_limit mentioned twice in the schema-lazr.conf.

# GitImportWorker.pushBazaarWorkingTree saves the git file from the

perhaps should say "git files"

# legacy:
and
# new hotness

As much as I like these, they don't fit our coding standards of full
sentences.

But other than that, great!

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/canonical/config/schema-lazr.conf'
2--- lib/canonical/config/schema-lazr.conf 2010-04-07 12:45:11 +0000
3+++ lib/canonical/config/schema-lazr.conf 2010-04-09 06:17:35 +0000
4@@ -440,9 +440,11 @@
5 # in a row.
6 consecutive_failure_limit: 5
7
8-# Import only this many revisions at once.
9-# Only applies to git imports for now.
10-revisions_import_limit: 1000
11+# Import only this many revisions from git at once.
12+git_revisions_import_limit: 10000
13+
14+# Import only this many revisions from svn (via bzr-svn) at once.
15+svn_revisions_import_limit: 1000
16
17
18 [codeimportdispatcher]
19
20=== modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py'
21--- lib/lp/codehosting/codeimport/tests/test_worker.py 2010-03-12 02:30:36 +0000
22+++ lib/lp/codehosting/codeimport/tests/test_worker.py 2010-04-09 06:17:35 +0000
23@@ -28,6 +28,7 @@
24 from canonical.testing import BaseLayer
25
26 from lp.codehosting import load_optional_plugin
27+from lp.codehosting.codeimport.tarball import create_tarball, extract_tarball
28 from lp.codehosting.codeimport.worker import (
29 BazaarBranchStore, BzrSvnImportWorker, CodeImportWorkerExitCode,
30 ForeignTreeStore, GitImportWorker, HgImportWorker, ImportDataStore,
31@@ -590,20 +591,23 @@
32 source_details, self.get_transport('import_data'),
33 self.makeBazaarBranchStore(), logging.getLogger("silent"))
34
35- def test_pushBazaarWorkingTree_saves_git_db(self):
36- # GitImportWorker.pushBazaarWorkingTree saves the git.db file from the
37- # tree's repository in the worker's ImportDataStore.
38+ def test_pushBazaarWorkingTree_saves_git_cache(self):
39+ # GitImportWorker.pushBazaarWorkingTree saves a tarball of the git
40+ # cache from the tree's repository in the worker's ImportDataStore.
41 content = self.factory.getUniqueString()
42 tree = self.make_branch_and_tree('.')
43- tree.branch.repository._transport.put_bytes('git.db', content)
44+ tree.branch.repository._transport.mkdir('git')
45+ tree.branch.repository._transport.put_bytes('git/cache', content)
46 import_worker = self.makeImportWorker()
47 import_worker.pushBazaarWorkingTree(tree)
48- import_worker.import_data_store.fetch('git.db')
49- self.assertEqual(content, open('git.db').read())
50+ import_worker.import_data_store.fetch('git-cache.tar.gz')
51+ extract_tarball('git-cache.tar.gz', '.')
52+ self.assertEqual(content, open('cache').read())
53
54- def test_getBazaarWorkingTree_fetches_git_db(self):
55- # GitImportWorker.getBazaarWorkingTree fetches the git.db file from
56- # the worker's ImportDataStore into the tree's repository.
57+ def test_getBazaarWorkingTree_fetches_legacy_git_db(self):
58+ # GitImportWorker.getBazaarWorkingTree fetches the legacy git.db file,
59+ # if present, from the worker's ImportDataStore into the tree's
60+ # repository.
61 import_worker = self.makeImportWorker()
62 # Store the git.db file in the store.
63 content = self.factory.getUniqueString()
64@@ -617,6 +621,25 @@
65 self.assertEqual(
66 content, tree.branch.repository._transport.get('git.db').read())
67
68+ def test_getBazaarWorkingTree_fetches_git_cache(self):
69+ # GitImportWorker.getBazaarWorkingTree fetches the tarball of the git
70+ # cache from the worker's ImportDataStore and expands it into the
71+ # tree's repository.
72+ import_worker = self.makeImportWorker()
73+ # Store a tarred-up cache in the store.x
74+ content = self.factory.getUniqueString()
75+ os.mkdir('cache')
76+ open('cache/git-cache', 'w').write(content)
77+ create_tarball('cache', 'git-cache.tar.gz')
78+ import_worker.import_data_store.put('git-cache.tar.gz')
79+ # Make sure there's a Bazaar branch in the branch store.
80+ tree = self.make_branch_and_tree('tree')
81+ ImportWorker.pushBazaarWorkingTree(import_worker, tree)
82+ # Finally, fetching the tree gets the git.db file too.
83+ tree = import_worker.getBazaarWorkingTree()
84+ self.assertEqual(
85+ content, tree.branch.repository._transport.get('git/git-cache').read())
86+
87
88 def clean_up_default_stores_for_import(source_details):
89 """Clean up the default branch and foreign tree stores for an import.
90@@ -913,7 +936,10 @@
91 self.makeForeignCommit(worker.source_details)
92 self.assertTrue(self.foreign_commit_count > 1)
93 self.pushConfig(
94- 'codeimport', revisions_import_limit=self.foreign_commit_count-1)
95+ 'codeimport',
96+ git_revisions_import_limit=self.foreign_commit_count-1,
97+ svn_revisions_import_limit=self.foreign_commit_count-1,
98+ )
99 self.assertEqual(
100 CodeImportWorkerExitCode.SUCCESS_PARTIAL, worker.run())
101 self.assertEqual(
102
103=== modified file 'lib/lp/codehosting/codeimport/worker.py'
104--- lib/lp/codehosting/codeimport/worker.py 2010-03-15 23:23:29 +0000
105+++ lib/lp/codehosting/codeimport/worker.py 2010-04-09 06:17:35 +0000
106@@ -26,7 +26,7 @@
107 from bzrlib.transport import get_transport
108 from bzrlib.errors import NoSuchFile, NotBranchError
109 import bzrlib.ui
110-from bzrlib.urlutils import join as urljoin
111+from bzrlib.urlutils import join as urljoin, local_path_from_url
112 from bzrlib.upgrade import upgrade
113
114 from canonical.cachedproperty import cachedproperty
115@@ -581,7 +581,7 @@
116
117 def getExtraPullArgs(self):
118 """See `PullingImportWorker.getExtraPullArgs`."""
119- return {'limit': config.codeimport.revisions_import_limit}
120+ return {'limit': config.codeimport.git_revisions_import_limit}
121
122 def getBazaarWorkingTree(self):
123 """See `ImportWorker.getBazaarWorkingTree`.
124@@ -591,8 +591,17 @@
125 it in the Bazaar tree, that is at '.bzr/repository/git.db'.
126 """
127 tree = PullingImportWorker.getBazaarWorkingTree(self)
128+ # Fetch the legacy cache from the store, if present.
129 self.import_data_store.fetch(
130 'git.db', tree.branch.repository._transport)
131+ # The cache dir from newer bzr-gits is stored as a tarball.
132+ local_name = 'git-cache.tar.gz'
133+ if self.import_data_store.fetch(local_name):
134+ repo_transport = tree.branch.repository._transport
135+ repo_transport.mkdir('git')
136+ git_db_dir = os.path.join(
137+ local_path_from_url(repo_transport.base), 'git')
138+ extract_tarball(local_name, git_db_dir)
139 return tree
140
141 def pushBazaarWorkingTree(self, bazaar_tree):
142@@ -604,8 +613,12 @@
143 """
144 non_trivial = PullingImportWorker.pushBazaarWorkingTree(
145 self, bazaar_tree)
146- self.import_data_store.put(
147- 'git.db', bazaar_tree.branch.repository._transport)
148+ repo_transport = bazaar_tree.branch.repository._transport
149+ git_db_dir = os.path.join(
150+ local_path_from_url(repo_transport.base), 'git')
151+ local_name = 'git-cache.tar.gz'
152+ create_tarball(git_db_dir, local_name)
153+ self.import_data_store.put(local_name)
154 return non_trivial
155
156
157@@ -655,7 +668,7 @@
158
159 def getExtraPullArgs(self):
160 """See `PullingImportWorker.getExtraPullArgs`."""
161- return {'limit': config.codeimport.revisions_import_limit}
162+ return {'limit': config.codeimport.svn_revisions_import_limit}
163
164 @property
165 def format_classes(self):
166
167=== modified file 'utilities/sourcedeps.conf'
168--- utilities/sourcedeps.conf 2010-04-07 12:27:00 +0000
169+++ utilities/sourcedeps.conf 2010-04-09 06:17:35 +0000
170@@ -1,10 +1,10 @@
171 bzr-builder lp:~launchpad-pqm/bzr-builder/trunk;revno=63
172-bzr-git lp:~launchpad-pqm/bzr-git/devel;revno=251
173+bzr-git lp:~launchpad-pqm/bzr-git/devel;revno=253
174 bzr-hg lp:~launchpad-pqm/bzr-hg/devel;revno=281
175 bzr-loom lp:~launchpad-pqm/bzr-loom/trunk;revno=47
176 bzr-svn lp:~launchpad-pqm/bzr-svn/devel;revno=2708
177 cscvs lp:~launchpad-pqm/launchpad-cscvs/devel;revno=430
178-dulwich lp:~launchpad-pqm/dulwich/devel;revno=417
179+dulwich lp:~launchpad-pqm/dulwich/devel;revno=418
180 launchpad-loggerhead lp:~launchpad-pqm/launchpad-loggerhead/devel;revno=54
181 loggerhead lp:~launchpad-pqm/loggerhead/devel;revno=174
182 lpreview lp:~launchpad-pqm/bzr-lpreview/devel;revno=23