Launchpad itself

Merge lp:~mwhudson/launchpad/no-hosted-area-join-launchpad-loggerhead into lp:launchpad/db-devel

  1. no-hosted-area-join-launchpad-loggerhead
  2. Merge into db-devel
Proposed by Michael Hudson-Doyle
Status: Rejected
Rejected by: Michael Hudson-Doyle
Proposed branch: lp:~mwhudson/launchpad/no-hosted-area-join-launchpad-loggerhead
Merge into: lp:launchpad/db-devel
Prerequisite: lp:~mwhudson/launchpad/no-hosted-area-server-catchup
Diff against target: 251 lines (+209/-4)
4 files modified
Makefile (+3/-3)
scripts/start-loggerhead.py (+175/-0)
scripts/stop-loggerhead.py (+31/-0)
utilities/sourcedeps.conf (+0/-1)
To merge this branch: bzr merge lp:~mwhudson/launchpad/no-hosted-area-join-launchpad-loggerhead
Reviewer Review Type Date Requested Status
Tim Penhey (community) Needs Information
Review via email: mp+24105@code.launchpad.net

Description of the change

Hi Tim,

This branch makes the launchpad-loggerhead code part of the launchpad tree, something I've been meaning to do for ages. No other changes.

Cheers,
mwh

To post a comment you must log in.
Revision history for this message
Tim Penhey (thumper) wrote :

scripts/start-loggerhead.py
 - Still says
#!/usr/bin/env python2.4
  instead of
#!/usr/bin/env python2.5

same for stop-loggerhead.py

Where is the content of the launchpad-loggerhead directory of the
sourcecode branch that has been removed? Did you forget to add it
in?

review: Needs Information

Unmerged revisions

9727. By Michael Hudson-Doyle

move the rest of the launchpad-loggerhead glue into the tree

9726. By Michael Hudson-Doyle

move launchpad_loggerhead code into lib

9725. By Michael Hudson-Doyle

join launchpad-loggerhead into the launchpad tree

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'Makefile'
2--- Makefile 2010-04-22 12:03:33 +0000
3+++ Makefile 2010-04-27 02:02:42 +0000
4@@ -229,13 +229,13 @@
5 -i $(LPCONFIG)
6
7 run_codebrowse: build
8- BZR_PLUGIN_PATH=bzrplugins $(PY) sourcecode/launchpad-loggerhead/start-loggerhead.py -f
9+ BZR_PLUGIN_PATH=bzrplugins $(PY) scripts/start-loggerhead.py -f
10
11 start_codebrowse: build
12- BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) sourcecode/launchpad-loggerhead/start-loggerhead.py
13+ BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) scripts/start-loggerhead.py
14
15 stop_codebrowse:
16- $(PY) sourcecode/launchpad-loggerhead/stop-loggerhead.py
17+ $(PY) scripts/stop-loggerhead.py
18
19 run_codehosting: check_schema inplace stop hosted_branches
20 $(RM) thread*.request
21
22=== removed directory 'lib/canonical/launchpad/apidoc'
23=== removed symlink 'lib/launchpad_loggerhead'
24=== target was u'../sourcecode/launchpad-loggerhead/launchpad_loggerhead/'
25=== added file 'scripts/start-loggerhead.py'
26--- scripts/start-loggerhead.py 1970-01-01 00:00:00 +0000
27+++ scripts/start-loggerhead.py 2010-04-27 02:02:42 +0000
28@@ -0,0 +1,175 @@
29+#!/usr/bin/env python2.4
30+#
31+# Copyright 2009 Canonical Ltd. This software is licensed under the
32+# GNU Affero General Public License version 3 (see the file LICENSE).
33+
34+import logging
35+import os
36+import sys
37+
38+from paste import httpserver
39+from paste.deploy.config import PrefixMiddleware
40+from paste.httpexceptions import HTTPExceptionHandler
41+from paste.request import construct_url
42+from paste.translogger import TransLogger
43+
44+from canonical.config import config
45+import lp.codehosting
46+
47+LISTEN_HOST = '0.0.0.0'
48+LISTEN_PORT = 8080
49+THREADPOOL_WORKERS = 10
50+
51+
52+class NoLockingFileHandler(logging.FileHandler):
53+ """A version of logging.FileHandler that doesn't do it's own locking.
54+
55+ We experienced occasional hangs in production where gdb-ery on the server
56+ revealed that we sometimes end up with many threads blocking on the RLock
57+ held by the logging file handler, and log reading finds that an exception
58+ managed to kill a thread in an unsafe window for RLock's.
59+
60+ Luckily, there's no real reason for us to take a lock during logging as
61+ each log message translates to one call to .write on a file object, which
62+ translates to one fwrite call, and it seems that this does enough locking
63+ itself for our purposes.
64+
65+ So this handler just doesn't lock in log message handling.
66+ """
67+
68+ def acquire(self):
69+ pass
70+
71+ def release(self):
72+ pass
73+
74+
75+def setup_logging(home, foreground):
76+ # i hate that stupid logging config format, so just set up logging here.
77+
78+ log_folder = config.codebrowse.log_folder
79+ if not log_folder:
80+ log_folder = os.path.join(home, 'logs')
81+ if not os.path.exists(log_folder):
82+ os.mkdir(log_folder)
83+
84+ f = logging.Formatter(
85+ '%(levelname)-.3s [%(asctime)s.%(msecs)03d] [%(thread)d] %(name)s: %(message)s',
86+ '%Y%m%d-%H:%M:%S')
87+ debug_log = NoLockingFileHandler(os.path.join(log_folder, 'debug.log'))
88+ debug_log.setLevel(logging.DEBUG)
89+ debug_log.setFormatter(f)
90+ if foreground:
91+ stdout_log = logging.StreamHandler(sys.stdout)
92+ stdout_log.setLevel(logging.DEBUG)
93+ stdout_log.setFormatter(f)
94+ f = logging.Formatter('[%(asctime)s.%(msecs)03d] %(message)s',
95+ '%Y%m%d-%H:%M:%S')
96+ access_log = NoLockingFileHandler(os.path.join(log_folder, 'access.log'))
97+ access_log.setLevel(logging.INFO)
98+ access_log.setFormatter(f)
99+
100+ logging.getLogger('').setLevel(logging.DEBUG)
101+ logging.getLogger('').addHandler(debug_log)
102+ logging.getLogger('wsgi').addHandler(access_log)
103+
104+ if foreground:
105+ logging.getLogger('').addHandler(stdout_log)
106+ else:
107+ class S(object):
108+ def write(self, str):
109+ logging.getLogger().error(str.rstrip('\n'))
110+ def flush(self):
111+ pass
112+ sys.stderr = S()
113+
114+
115+
116+foreground = False
117+if len(sys.argv) > 1:
118+ if sys.argv[1] == '-f':
119+ foreground = True
120+
121+home = os.path.realpath(os.path.dirname(__file__))
122+pidfile = os.path.join(home, 'loggerhead.pid')
123+
124+if not foreground:
125+ sys.stderr.write('\n')
126+ sys.stderr.write('Launching loggerhead into the background.\n')
127+ sys.stderr.write('PID file: %s\n' % (pidfile,))
128+ sys.stderr.write('\n')
129+
130+ from loggerhead.daemon import daemonize
131+ daemonize(pidfile, home)
132+
133+setup_logging(home, foreground=foreground)
134+
135+log = logging.getLogger('loggerhead')
136+log.info('Starting up...')
137+
138+log.info('Loading the bzr plugins...')
139+from bzrlib.plugin import load_plugins
140+load_plugins()
141+
142+import bzrlib.plugins
143+if getattr(bzrlib.plugins, 'loom', None) is None:
144+ log.error('Loom plugin loading failed.')
145+
146+from launchpad_loggerhead.debug import (
147+ change_kill_thread_criteria, threadpool_debug)
148+from launchpad_loggerhead.app import RootApp
149+from launchpad_loggerhead.session import SessionHandler
150+
151+SESSION_VAR = 'lh.session'
152+
153+secret = open(os.path.join(config.root, config.codebrowse.secret_path)).read()
154+
155+app = RootApp(SESSION_VAR)
156+app = HTTPExceptionHandler(app)
157+app = SessionHandler(app, SESSION_VAR, secret)
158+def log_on_request_start(app):
159+ def wrapped(environ, start_response):
160+ log = logging.getLogger('loggerhead')
161+ log.info("Starting to process %s", construct_url(environ))
162+ return app(environ, start_response)
163+ return wrapped
164+app = log_on_request_start(app)
165+app = PrefixMiddleware(app)
166+app = TransLogger(app)
167+app = threadpool_debug(app)
168+
169+def set_scheme(app):
170+ """Set wsgi.url_scheme in the environment correctly.
171+
172+ We serve requests that originated from both http and https, and
173+ distinguish between them by adding a header in the https Apache config.
174+ """
175+ def wrapped(environ, start_response):
176+ environ['wsgi.url_scheme'] = environ.pop(
177+ 'HTTP_X_FORWARDED_SCHEME', 'http')
178+ return app(environ, start_response)
179+ return wrapped
180+app = set_scheme(app)
181+app = change_kill_thread_criteria(app)
182+
183+try:
184+ httpserver.serve(
185+ app, host=LISTEN_HOST, port=LISTEN_PORT,
186+ threadpool_workers=THREADPOOL_WORKERS,
187+ threadpool_options={
188+ # Kill threads after 300 seconds. This is insanely high, but
189+ # lower enough than the default (1800 seconds!) that evidence
190+ # suggests it will be hit occasionally, and there's very little
191+ # chance of it having negative consequences.
192+ 'kill_thread_limit': 300,
193+ # Check for threads that should be killed every 10 requests. The
194+ # default is every 100, which is easily long enough for things to
195+ # gum up completely in between checks.
196+ 'hung_check_period': 10,
197+ })
198+finally:
199+ log.info('Shutdown.')
200+ try:
201+ os.remove(pidfile)
202+ except OSError:
203+ pass
204
205=== added file 'scripts/stop-loggerhead.py'
206--- scripts/stop-loggerhead.py 1970-01-01 00:00:00 +0000
207+++ scripts/stop-loggerhead.py 2010-04-27 02:02:42 +0000
208@@ -0,0 +1,31 @@
209+#!/usr/bin/env python2.4
210+#
211+# Copyright 2009 Canonical Ltd. This software is licensed under the
212+# GNU Affero General Public License version 3 (see the file LICENSE).
213+
214+import os
215+import sys
216+
217+home = os.path.realpath(os.path.dirname(__file__))
218+pidfile = os.path.join(home, 'loggerhead.pid')
219+
220+try:
221+ f = open(pidfile, 'r')
222+except IOError, e:
223+ print 'No pid file found.'
224+ sys.exit(1)
225+
226+pid = int(f.readline())
227+
228+try:
229+ os.kill(pid, 0)
230+except OSError, e:
231+ print 'Stale pid file; server is not running.'
232+ sys.exit(1)
233+
234+print
235+print 'Shutting down previous server @ pid %d.' % (pid,)
236+print
237+
238+import signal
239+os.kill(pid, signal.SIGTERM)
240
241=== modified file 'utilities/sourcedeps.conf'
242--- utilities/sourcedeps.conf 2010-04-21 12:30:48 +0000
243+++ utilities/sourcedeps.conf 2010-04-27 02:02:42 +0000
244@@ -5,7 +5,6 @@
245 bzr-svn lp:~launchpad-pqm/bzr-svn/devel;revno=2708
246 cscvs lp:~launchpad-pqm/launchpad-cscvs/devel;revno=432
247 dulwich lp:~launchpad-pqm/dulwich/devel;revno=418
248-launchpad-loggerhead lp:~launchpad-pqm/launchpad-loggerhead/devel;revno=54
249 loggerhead lp:~launchpad-pqm/loggerhead/devel;revno=174
250 lpreview lp:~launchpad-pqm/bzr-lpreview/devel;revno=23
251 mailman lp:~launchpad-pqm/mailman/2.1;revno=976

Subscribers

People subscribed via source and target branches

to status/vote changes: