Launchpad itself

Merge lp:~jml/launchpad/lp-poppy into lp:launchpad

  1. lp-poppy
  2. Merge into devel
Proposed by Jonathan Lange
Status: Merged
Approved by: Julian Edwards
Approved revision: no longer in the source branch.
Merged at revision: not available
Proposed branch: lp:~jml/launchpad/lp-poppy
Merge into: lp:launchpad
Diff against target: 377 lines (+83/-107)
10 files modified
daemons/poppy-upload.py (+1/-48)
lib/lp/archiveuploader/uploadprocessor.py (+4/-4)
lib/lp/poppy/daemon.py (+55/-0)
lib/lp/poppy/hooks.py (+6/-4)
lib/lp/poppy/server.py (+4/-41)
lib/lp/poppy/tests/filesystem.txt (+1/-1)
lib/lp/poppy/tests/test_poppy.py (+2/-1)
lib/lp/soyuz/doc/soyuz-upload.txt.disabled (+1/-2)
lib/lp/soyuz/interfaces/queue.py (+4/-3)
lib/lp/soyuz/model/queue.py (+5/-3)
To merge this branch: bzr merge lp:~jml/launchpad/lp-poppy
Reviewer Review Type Date Requested Status
Julian Edwards (community) Approve
Review via email: mp+21551@code.launchpad.net

Commit message

Move poppy code to lp.poppy.

Description of the change

This branch moves all the poppy code into one place: lp.poppy. In particular, it moves lp.archiveuploader.poppyinterface to lp.poppy.hooks, and most of the actual code in daemons/poppy-upload.py to lp.poppy.daemon.

It also deletes a bunch of unused code and removes as many mentions of "lucille" as make sense to remove.

To post a comment you must log in.
Revision history for this message
Julian Edwards (julian-edwards) wrote :

Thank you for making Poppy nicer :)

Remove the comments:

+ # host = "127.0.0.1"
+ # host = "82.211.81.167" # Drescher's public IP

as discussed and it's good to land.

Cheers!

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'daemons/poppy-upload.py'
--- daemons/poppy-upload.py 2009-10-17 14:06:03 +0000
+++ daemons/poppy-upload.py 2010-03-17 12:28:38 +0000
@@ -4,54 +4,7 @@
4# GNU Affero General Public License version 3 (see the file LICENSE).4# GNU Affero General Public License version 3 (see the file LICENSE).
55
6import sys6import sys
7import logging7from lp.poppy.daemon import main
8import optparse
9
10from canonical.poppy.server import run_server
11from lp.archiveuploader.poppyinterface import PoppyInterface
12from canonical.launchpad.scripts import logger, logger_options
13
14
15def main():
16
17 parser = optparse.OptionParser()
18 logger_options(parser)
19
20 parser.add_option("--cmd", action="store", metavar="CMD",
21 help="Run CMD after each upload completion")
22
23 parser.add_option("--allow-user", action="store", metavar="USER",
24 default='ubuntu',
25 help="Username allowed to log in.")
26
27 parser.add_option("--permissions", action="store", metavar="PERMS",
28 default='g+rwxs',
29 help="Permissions to chmod the targetfsroot with "
30 "before letting go of the directory.")
31
32 options, args = parser.parse_args()
33
34 log = logger(options, "poppy-upload")
35
36 if len(args) != 2:
37 print "usage: poppy-upload.py rootuploaddirectory port"
38 return 1
39
40 root, port = args
41 # host = "127.0.0.1"
42 # host = "82.211.81.167" # Drescher's public IP
43 host = "0.0.0.0"
44 ident = "lucille upload server"
45 numthreads = 4
46
47 iface = PoppyInterface(root, log, allow_user=options.allow_user,
48 cmd=options.cmd,
49 perms=options.permissions)
50
51 run_server(host, int(port), ident, numthreads,
52 iface.new_client_hook, iface.client_done_hook,
53 iface.auth_verify_hook)
54 return 0
558
56if __name__ == '__main__':9if __name__ == '__main__':
57 sys.exit(main())10 sys.exit(main())
5811
=== modified file 'lib/lp/archiveuploader/uploadprocessor.py'
--- lib/lp/archiveuploader/uploadprocessor.py 2010-02-05 10:54:07 +0000
+++ lib/lp/archiveuploader/uploadprocessor.py 2010-03-17 12:28:38 +0000
@@ -218,9 +218,9 @@
218 :return: a list of upload directories found in the queue218 :return: a list of upload directories found in the queue
219 alphabetically sorted.219 alphabetically sorted.
220 """220 """
221 # Protecting listdir by a lock ensures that we only get221 # Protecting listdir by a lock ensures that we only get completely
222 # completely finished directories listed. See222 # finished directories listed. See lp.poppy.hooks for the other
223 # PoppyInterface for the other locking place.223 # locking place.
224 lockfile_path = os.path.join(fsroot, ".lock")224 lockfile_path = os.path.join(fsroot, ".lock")
225 fsroot_lock = GlobalLock(lockfile_path)225 fsroot_lock = GlobalLock(lockfile_path)
226 mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)226 mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
@@ -239,7 +239,7 @@
239 fsroot_lock.acquire(blocking=True)239 fsroot_lock.acquire(blocking=True)
240 dir_names = os.listdir(fsroot)240 dir_names = os.listdir(fsroot)
241 finally:241 finally:
242 # Skip lockfile deletion, see similar code in poppyinterface.py.242 # Skip lockfile deletion, see similar code in lp.poppy.hooks.
243 fsroot_lock.release(skip_delete=True)243 fsroot_lock.release(skip_delete=True)
244244
245 sorted_dir_names = sorted(245 sorted_dir_names = sorted(
246246
=== renamed directory 'lib/canonical/poppy' => 'lib/lp/poppy'
=== added file 'lib/lp/poppy/daemon.py'
--- lib/lp/poppy/daemon.py 1970-01-01 00:00:00 +0000
+++ lib/lp/poppy/daemon.py 2010-03-17 12:28:38 +0000
@@ -0,0 +1,55 @@
1# Copyright 2010 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""The Poppy daemon."""
5
6__metaclass__ = type
7__all__ = [
8 'main',
9 ]
10
11import optparse
12
13from canonical.launchpad.scripts import logger, logger_options
14from lp.poppy.hooks import Hooks
15from lp.poppy.server import run_server
16
17
18def main():
19 parser = optparse.OptionParser()
20 logger_options(parser)
21
22 parser.add_option("--cmd", action="store", metavar="CMD",
23 help="Run CMD after each upload completion")
24
25 parser.add_option("--allow-user", action="store", metavar="USER",
26 default='ubuntu',
27 help="Username allowed to log in.")
28
29 parser.add_option("--permissions", action="store", metavar="PERMS",
30 default='g+rwxs',
31 help="Permissions to chmod the targetfsroot with "
32 "before letting go of the directory.")
33
34 options, args = parser.parse_args()
35
36 log = logger(options, "poppy-upload")
37
38 if len(args) != 2:
39 print "usage: poppy-upload.py rootuploaddirectory port"
40 return 1
41
42 root, port = args
43 host = "0.0.0.0"
44 ident = "lucille upload server"
45 numthreads = 4
46
47 hooks = Hooks(root, log, allow_user=options.allow_user, cmd=options.cmd,
48 perms=options.permissions)
49
50 run_server(host, int(port), ident, numthreads,
51 hooks.new_client_hook, hooks.client_done_hook,
52 hooks.auth_verify_hook)
53 return 0
54
55
056
=== renamed file 'lib/lp/archiveuploader/poppyinterface.py' => 'lib/lp/poppy/hooks.py'
--- lib/lp/archiveuploader/poppyinterface.py 2009-11-13 15:29:46 +0000
+++ lib/lp/poppy/hooks.py 2010-03-17 12:28:38 +0000
@@ -1,7 +1,7 @@
1# Copyright 2009 Canonical Ltd. This software is licensed under the1# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4# Lucille's primary interface to the upload mechanism4__metaclass__ = type
55
6import logging6import logging
7import shutil7import shutil
@@ -11,17 +11,19 @@
1111
12from contrib.glock import GlobalLock12from contrib.glock import GlobalLock
1313
14
14class PoppyInterfaceFailure(Exception):15class PoppyInterfaceFailure(Exception):
15 pass16 pass
1617
17class PoppyInterface:18
19class Hooks:
1820
19 clients = {}21 clients = {}
2022
21 def __init__(self, targetpath, logger, allow_user, cmd=None,23 def __init__(self, targetpath, logger, allow_user, cmd=None,
22 targetstart=0, perms=None):24 targetstart=0, perms=None):
23 self.targetpath = targetpath25 self.targetpath = targetpath
24 self.logger = logging.getLogger("%s.PoppyInterface" % logger.name)26 self.logger = logging.getLogger("%s.Hooks" % logger.name)
25 self.cmd = cmd27 self.cmd = cmd
26 self.allow_user = allow_user28 self.allow_user = allow_user
27 self.targetcount = targetstart29 self.targetcount = targetstart
2830
=== modified file 'lib/lp/poppy/server.py'
--- lib/canonical/poppy/server.py 2009-08-31 18:03:05 +0000
+++ lib/lp/poppy/server.py 2010-03-17 12:28:38 +0000
@@ -4,8 +4,6 @@
4__metaclass__ = type4__metaclass__ = type
55
6import asyncore6import asyncore
7import logging
8import sys
9import tempfile7import tempfile
10from time import time8from time import time
119
@@ -14,7 +12,7 @@
14from zope.server.taskthreads import ThreadedTaskDispatcher12from zope.server.taskthreads import ThreadedTaskDispatcher
15from zope.server.serverbase import ServerBase13from zope.server.serverbase import ServerBase
1614
17from canonical.poppy.filesystem import UploadFileSystem15from lp.poppy.filesystem import UploadFileSystem
1816
1917
20class Channel(FTPServerChannel):18class Channel(FTPServerChannel):
@@ -39,7 +37,7 @@
39 return self.uploadfilesystem37 return self.uploadfilesystem
4038
41 def received(self, data):39 def received(self, data):
42 # XXX Steve Alexander 2005-01-18 40 # XXX Steve Alexander 2005-01-18
43 # This is a work-around for a bug in Zope 3's ServerChannelBase41 # This is a work-around for a bug in Zope 3's ServerChannelBase
44 # that it doesn't update self.last_activity.42 # that it doesn't update self.last_activity.
45 # This method can be removed once Zope3 is fixed, and we're using43 # This method can be removed once Zope3 is fixed, and we're using
@@ -112,6 +110,7 @@
112 # This is the point at which some data for an upload has been110 # This is the point at which some data for an upload has been
113 # received by the server from a client.111 # received by the server from a client.
114112
113
115class Server(ServerBase):114class Server(ServerBase):
116115
117 channel_class = Channel116 channel_class = Channel
@@ -145,7 +144,7 @@
145144
146145
147def run_server(host, port, ident, numthreads,146def run_server(host, port, ident, numthreads,
148 new_client_hook, client_done_hook, auth_verify_hook = None):147 new_client_hook, client_done_hook, auth_verify_hook=None):
149 task_dispatcher = ThreadedTaskDispatcher()148 task_dispatcher = ThreadedTaskDispatcher()
150 task_dispatcher.setThreadCount(numthreads)149 task_dispatcher.setThreadCount(numthreads)
151 server = Server(host, port,150 server = Server(host, port,
@@ -157,39 +156,3 @@
157 except KeyboardInterrupt:156 except KeyboardInterrupt:
158 # Exit without spewing an exception.157 # Exit without spewing an exception.
159 pass158 pass
160
161
162def main():
163 args = sys.argv[1:]
164 if len(args) != 1:
165 print "usage: server.py port"
166 return 1
167 port = int(args[0])
168 host = "127.0.0.1"
169 ident = "lucille upload server"
170 numthreads = 4
171
172 logger = logging.getLogger('Server')
173 hdlr = logging.FileHandler('+lucilleupload.log')
174 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
175 hdlr.setFormatter(formatter)
176 logger.addHandler(hdlr)
177 logger.setLevel(logging.WARNING)
178
179 def new_client_hook(fsroot, host, port):
180 print "new client:", fsroot, host, port
181
182 def client_done_hook(fsroot, host, port):
183 print "client done:", fsroot, host, port
184
185 def auth_verify_hook(fsroot, user, passw):
186 print "Auth Verification hook:", fsroot, user, passw
187 return True
188
189 run_server(host, int(port), ident, numthreads,
190 new_client_hook, client_done_hook,
191 auth_verify_hook)
192 return 0
193
194if __name__ == '__main__':
195 sys.exit(main())
196159
=== modified file 'lib/lp/poppy/tests/filesystem.txt'
--- lib/canonical/poppy/tests/filesystem.txt 2009-10-15 12:44:02 +0000
+++ lib/lp/poppy/tests/filesystem.txt 2010-03-17 12:28:38 +0000
@@ -2,7 +2,7 @@
2This is an implementation of IFileSystem which the FTP Server in Zope32This is an implementation of IFileSystem which the FTP Server in Zope3
3uses to know what to do when people make FTP commands.3uses to know what to do when people make FTP commands.
44
5 >>> from canonical.poppy.filesystem import UploadFileSystem5 >>> from lp.poppy.filesystem import UploadFileSystem
66
7The UploadFileSystem class implements the interface IFileSystem.7The UploadFileSystem class implements the interface IFileSystem.
88
99
=== modified file 'lib/lp/poppy/tests/test_poppy.py'
--- lib/canonical/poppy/tests/test_poppy.py 2009-06-25 05:30:52 +0000
+++ lib/lp/poppy/tests/test_poppy.py 2010-03-17 12:28:38 +0000
@@ -13,8 +13,9 @@
13import unittest13import unittest
14import StringIO14import StringIO
1515
16from canonical.poppy.tests import PoppyTestSetup
17from canonical.testing import LaunchpadZopelessLayer16from canonical.testing import LaunchpadZopelessLayer
17from lp.poppy.tests import PoppyTestSetup
18
1819
19class TestPoppy(unittest.TestCase):20class TestPoppy(unittest.TestCase):
20 """Test if poppy.py daemon works properly."""21 """Test if poppy.py daemon works properly."""
2122
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt.disabled'
--- lib/lp/soyuz/doc/soyuz-upload.txt.disabled 2010-02-15 12:59:55 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt.disabled 2010-03-17 12:28:38 +0000
@@ -41,8 +41,7 @@
41call the upload processing tool. We'll do that ourselves in our test,41call the upload processing tool. We'll do that ourselves in our test,
42so that we can control what's going on.42so that we can control what's going on.
4343
44 >>> from canonical.poppy.tests import (44 >>> from lp.poppy.tests import PoppyTestSetup, SoyuzUploadError
45 ... PoppyTestSetup, SoyuzUploadError)
46 >>> poppy = PoppyTestSetup(incoming_dir)45 >>> poppy = PoppyTestSetup(incoming_dir)
47 >>> poppy.startPoppy()46 >>> poppy.startPoppy()
4847
4948
=== modified file 'lib/lp/soyuz/interfaces/queue.py'
--- lib/lp/soyuz/interfaces/queue.py 2010-03-08 12:00:29 +0000
+++ lib/lp/soyuz/interfaces/queue.py 2010-03-17 12:28:38 +0000
@@ -139,7 +139,8 @@
139139
140140
141class IPackageUpload(Interface):141class IPackageUpload(Interface):
142 """A Queue item for Lucille"""142 """A Queue item for the archive uploader."""
143
143 export_as_webservice_entry()144 export_as_webservice_entry()
144145
145 id = Int(146 id = Int(
@@ -395,7 +396,7 @@
395396
396397
397class IPackageUploadBuild(Interface):398class IPackageUploadBuild(Interface):
398 """A Queue item's related builds (for Lucille)"""399 """A Queue item's related builds."""
399400
400 id = Int(401 id = Int(
401 title=_("ID"), required=True, readonly=True,402 title=_("ID"), required=True, readonly=True,
@@ -431,7 +432,7 @@
431 """432 """
432433
433class IPackageUploadSource(Interface):434class IPackageUploadSource(Interface):
434 """A Queue item's related sourcepackagereleases (for Lucille)"""435 """A Queue item's related sourcepackagereleases."""
435436
436 id = Int(437 id = Int(
437 title=_("ID"), required=True, readonly=True,438 title=_("ID"), required=True, readonly=True,
438439
=== modified file 'lib/lp/soyuz/model/queue.py'
--- lib/lp/soyuz/model/queue.py 2010-03-15 10:22:34 +0000
+++ lib/lp/soyuz/model/queue.py 2010-03-17 12:28:38 +0000
@@ -132,7 +132,8 @@
132132
133133
134class PackageUpload(SQLBase):134class PackageUpload(SQLBase):
135 """A Queue item for Lucille."""135 """A Queue item for the archive uploader."""
136
136 implements(IPackageUpload)137 implements(IPackageUpload)
137138
138 _defaultOrder = ['id']139 _defaultOrder = ['id']
@@ -1349,7 +1350,7 @@
13491350
13501351
1351class PackageUploadBuild(SQLBase):1352class PackageUploadBuild(SQLBase):
1352 """A Queue item's related builds (for Lucille)."""1353 """A Queue item's related builds."""
1353 implements(IPackageUploadBuild)1354 implements(IPackageUploadBuild)
13541355
1355 _defaultOrder = ['id']1356 _defaultOrder = ['id']
@@ -1449,7 +1450,8 @@
14491450
14501451
1451class PackageUploadSource(SQLBase):1452class PackageUploadSource(SQLBase):
1452 """A Queue item's related sourcepackagereleases (for Lucille)."""1453 """A Queue item's related sourcepackagereleases."""
1454
1453 implements(IPackageUploadSource)1455 implements(IPackageUploadSource)
14541456
1455 _defaultOrder = ['id']1457 _defaultOrder = ['id']