1
=== modified file '.bzrignore'
2
--- .bzrignore	2009-09-10 06:11:21 +0000
3
+++ .bzrignore	2010-05-05 01:58:48 +0000
4
@@ -30,11 +30,12 @@
5
30
database/sampledata/newsampledata-dev.sql
30
database/sampledata/newsampledata-dev.sql
6
31
database/sampledata/lintdata.sql
31
database/sampledata/lintdata.sql
7
32
database/sampledata/lintdata-dev.sql
32
database/sampledata/lintdata-dev.sql
9
33
lib/canonical/launchpad/apidoc/index.html
33
lib/canonical/launchpad/apidoc/*.html
10
34
xxx-report.*
34
xxx-report.*
12
35
lib/canonical/launchpad/apidoc/wadl-development.xml
35
lib/canonical/launchpad/apidoc/wadl-development-*.xml
13
36
lib/canonical/launchpad/apidoc/wadl-test-playground.xml
36
lib/canonical/launchpad/apidoc/wadl-test-playground.xml
14
37
lib/canonical/launchpad/icing/build/*
37
lib/canonical/launchpad/icing/build/*
15
38
lib/canonical/launchpad/icing/combo.css
16
38
trace.log
39
trace.log
17
39
test-appserver-layer-trace.log
40
test-appserver-layer-trace.log
18
40
bin
41
bin
19
@@ -51,3 +52,17 @@
20
51
./production-configs
52
./production-configs
21
52
bzr.dev
53
bzr.dev
22
53
_trial_temp
54
_trial_temp
23
55
lazr-js
24
56
.bazaar
25
57
.cache
26
58
.subversion
27
59
lib/canonical/buildd/launchpad-files
28
60
.testrepository
29
61
.memcache.pid
30
62
./pipes
31
63
lib/canonical/launchpad/apidoc/wadl-development-*.xml
32
64
tags.new
33
65
lp-clustered.dot
34
66
lp-clustered.sfood
35
67
lp-clustered.svg
36
68
lp.sfood
37
54
69
38
=== added file '.ctags'
39
--- .ctags	1970-01-01 00:00:00 +0000
40
+++ .ctags	2010-05-05 01:58:48 +0000
41
@@ -0,0 +1,1 @@
42
1
--python-kinds=-iv
43
0
2
44
=== added file '.testr.conf'
45
--- .testr.conf	1970-01-01 00:00:00 +0000
46
+++ .testr.conf	2010-05-05 01:58:48 +0000
47
@@ -0,0 +1,3 @@
48
1
[DEFAULT]
49
2
test_command=./bin/test --subunit $IDOPTION
50
3
test_id_option=--load-list $IDFILE
51
0
4
52
=== modified file 'Makefile'
53
--- Makefile	2009-10-07 09:55:24 +0000
54
+++ Makefile	2010-05-05 01:58:48 +0000
55
@@ -1,7 +1,7 @@
56
1
# This file modified from Zope3/Makefile
1
# This file modified from Zope3/Makefile
57
2
# Licensed under the ZPL, (c) Zope Corporation and contributors.
2
# Licensed under the ZPL, (c) Zope Corporation and contributors.
58
3
3
60
4
PYTHON_VERSION=2.4
4
PYTHON_VERSION=2.5
61
5
PYTHON=python${PYTHON_VERSION}
5
PYTHON=python${PYTHON_VERSION}
62
6
WD:=$(shell pwd)
6
WD:=$(shell pwd)
63
7
PY=$(WD)/bin/py
7
PY=$(WD)/bin/py
64
@@ -17,18 +17,34 @@
65
17
17
66
18
LPCONFIG=development
18
LPCONFIG=development
67
19
19
68
20
JSFLAGS=
69
21
ICING=lib/canonical/launchpad/icing
70
22
LP_BUILT_JS_ROOT=${ICING}/build
71
23
LAZR_BUILT_JS_ROOT=lazr-js/build
72
24
73
20
MINS_TO_SHUTDOWN=15
25
MINS_TO_SHUTDOWN=15
74
21
26
75
22
CODEHOSTING_ROOT=/var/tmp/bazaar.launchpad.dev
27
CODEHOSTING_ROOT=/var/tmp/bazaar.launchpad.dev
76
23
28
77
24
BZR_VERSION_INFO = bzr-version-info.py
29
BZR_VERSION_INFO = bzr-version-info.py
78
25
30
81
26
WADL_FILE = lib/canonical/launchpad/apidoc/wadl-$(LPCONFIG).xml
31
APIDOC_DIR = lib/canonical/launchpad/apidoc
82
27
API_INDEX = lib/canonical/launchpad/apidoc/index.html
32
WADL_TEMPLATE = $(APIDOC_DIR).tmp/wadl-$(LPCONFIG)-%(version)s.xml
83
33
API_INDEX = $(APIDOC_DIR)/index.html
84
28
34
88
29
EXTRA_JS_FILES=lib/canonical/launchpad/icing/MochiKit.js \
35
# Do not add bin/buildout to this list.
89
30
				$(shell $(HERE)/utilities/yui-deps.py) \
36
# It is impossible to get buildout to tell us all the files it would
90
31
				lib/canonical/launchpad/icing/lazr/build/lazr.js
37
# build, since each egg's setup.py doesn't tell us that information.
91
38
BUILDOUT_BIN = \
92
39
    $(PY) bin/apiindex bin/combine-css bin/fl-build-report \
93
40
    bin/fl-credential-ctl bin/fl-install-demo bin/fl-monitor-ctl \
94
41
    bin/fl-record bin/fl-run-bench bin/fl-run-test bin/googletestservice \
95
42
    bin/i18ncompile bin/i18nextract bin/i18nmergeall bin/i18nstats \
96
43
    bin/harness bin/iharness bin/ipy bin/jsbuild bin/jslint bin/jssize \
97
44
    bin/jstest bin/killservice bin/kill-test-services bin/lint.sh \
98
45
    bin/lp-windmill bin/retest bin/run bin/sprite-util \
99
46
    bin/start_librarian bin/stxdocs bin/tags bin/test bin/tracereport \
100
47
    bin/twistd bin/update-download-cache bin/windmill
101
32
48
102
33
# DO NOT ALTER : this should just build by default
49
# DO NOT ALTER : this should just build by default
103
34
default: inplace
50
default: inplace
104
@@ -43,22 +59,14 @@
105
43
hosted_branches: $(PY)
59
hosted_branches: $(PY)
106
44
	$(PY) ./utilities/make-dummy-hosted-branches
60
	$(PY) ./utilities/make-dummy-hosted-branches
107
45
61
115
46
$(WADL_FILE): $(BZR_VERSION_INFO)
62
$(API_INDEX): $(BZR_VERSION_INFO)
116
47
	LPCONFIG=$(LPCONFIG) $(PY) ./utilities/create-lp-wadl.py > $@.tmp
63
	mkdir -p $(APIDOC_DIR).tmp
117
48
	mv $@.tmp $@
64
	LPCONFIG=$(LPCONFIG) $(PY) ./utilities/create-lp-wadl-and-apidoc.py "$(WADL_TEMPLATE)"
118
49
65
	mv $(APIDOC_DIR).tmp/* $(APIDOC_DIR)
119
50
$(API_INDEX): $(WADL_FILE)
66
	rmdir $(APIDOC_DIR).tmp
113
51
	bin/apiindex $(WADL_FILE) > $@.tmp
114
52
	mv $@.tmp $@
120
53
67
121
54
apidoc: compile $(API_INDEX)
68
apidoc: compile $(API_INDEX)
122
55
69
123
56
check_loggerhead_on_merge:
124
57
	# Loggerhead doesn't depend on anything else in rocketfuel and nothing
125
58
	# depends on it (yet).
126
59
	make -C sourcecode/loggerhead check PYTHON=${PYTHON} \
127
60
		PYTHON_VERSION=${PYTHON_VERSION} PYTHONPATH=$(PYTHONPATH)
128
61
129
62
check_merge: $(PY)
70
check_merge: $(PY)
130
63
	[ `PYTHONPATH= bzr status -S database/schema/ | \
71
	[ `PYTHONPATH= bzr status -S database/schema/ | \
131
64
		grep -v "\(^P\|pending\|security.cfg\|Makefile\|unautovacuumable\|_pythonpath.py\)" | wc -l` -eq 0 ]
72
		grep -v "\(^P\|pending\|security.cfg\|Makefile\|unautovacuumable\|_pythonpath.py\)" | wc -l` -eq 0 ]
132
@@ -67,16 +75,12 @@
133
67
check_db_merge: $(PY)
75
check_db_merge: $(PY)
134
68
	${PY} lib/canonical/tests/test_no_conflict_marker.py
76
	${PY} lib/canonical/tests/test_no_conflict_marker.py
135
69
77
136
70
# This can be removed once we move to zc.buildout and we have versioned
137
71
# dependencies, but for now we run both Launchpad and all other
138
72
# dependencies tests for any merge to sourcecode.
139
73
check_sourcecode_merge: check
140
74
	$(MAKE) -C sourcecode check PYTHON=${PYTHON} \
141
75
		PYTHON_VERSION=${PYTHON_VERSION} PYTHONPATH=$(PYTHONPATH)
142
76
143
77
check_config: build
78
check_config: build
144
78
	bin/test -m canonical.config.tests -vvt test_config
79
	bin/test -m canonical.config.tests -vvt test_config
145
79
80
146
81
check_schema: build
147
82
	${PY} utilities/check-db-revision.py
148
83
149
80
# Clean before running the test suite, since the build might fail depending
84
# Clean before running the test suite, since the build might fail depending
150
81
# what source changes happened. (e.g. apidoc depends on interfaces)
85
# what source changes happened. (e.g. apidoc depends on interfaces)
151
82
check: clean build
86
check: clean build
152
@@ -90,9 +94,7 @@
153
90
	@echo
94
	@echo
154
91
	@echo "Running the JavaScript integration test suite"
95
	@echo "Running the JavaScript integration test suite"
155
92
	@echo
96
	@echo
159
93
	bin/test $(VERBOSITY) --layer=BugsWindmillLayer
97
	bin/test $(VERBOSITY) --layer=WindmillLayer
157
94
	bin/test $(VERBOSITY) --layer=CodeWindmillLayer
158
95
	bin/test $(VERBOSITY) --layer=SoyuzWindmillLayer
160
96
98
161
97
jscheck_functest: build
99
jscheck_functest: build
162
98
    # Run the old functest Windmill integration tests.  The test runner
100
    # Run the old functest Windmill integration tests.  The test runner
163
@@ -124,37 +126,76 @@
164
124
126
165
125
inplace: build
127
inplace: build
166
126
128
168
127
build: $(BZR_VERSION_INFO) compile apidoc
129
build: compile apidoc jsbuild css_combine
169
130
170
131
css_combine: sprite_css bin/combine-css
171
132
	${SHHH} bin/combine-css
172
133
173
134
sprite_css: ${LP_BUILT_JS_ROOT}/style-3-0.css
174
135
175
136
${LP_BUILT_JS_ROOT}/style-3-0.css: bin/sprite-util ${ICING}/style-3-0.css.in ${ICING}/icon-sprites.positioning
176
137
	${SHHH} bin/sprite-util create-css
177
138
178
139
sprite_image:
179
140
	${SHHH} bin/sprite-util create-image
180
141
181
142
jsbuild_lazr: bin/jsbuild
182
143
	# We absolutely do not want to include the lazr.testing module and its
183
144
	# jsTestDriver test harness modifications in the lazr.js and launchpad.js
184
145
	# roll-up files.  They fiddle with built-in functions!  See Bug 482340.
185
146
	${SHHH} bin/jsbuild $(JSFLAGS) -b $(LAZR_BUILT_JS_ROOT) -x testing/ -c $(LAZR_BUILT_JS_ROOT)/yui
186
147
187
148
jsbuild: jsbuild_lazr bin/jsbuild bin/jssize
188
149
	${SHHH} bin/jsbuild \
189
150
		$(JSFLAGS) \
190
151
		-n launchpad \
191
152
		-s lib/canonical/launchpad/javascript \
192
153
		-b $(LP_BUILT_JS_ROOT) \
193
154
		$(shell $(HERE)/utilities/yui-deps.py) \
194
155
		$(shell $(HERE)/utilities/lp-deps.py) \
195
156
		lib/canonical/launchpad/icing/lazr/build/lazr.js
196
157
	${SHHH} bin/jssize
197
128
158
198
129
eggs:
159
eggs:
199
130
	# Usually this is linked via link-external-sourcecode, but in
160
	# Usually this is linked via link-external-sourcecode, but in
200
131
	# deployment we create this ourselves.
161
	# deployment we create this ourselves.
201
132
	mkdir eggs
162
	mkdir eggs
202
133
163
203
164
# LP_SOURCEDEPS_PATH should point to the sourcecode directory, but we
204
165
# want the parent directory where the download-cache and eggs directory
205
166
# are. We re-use the variable that is using for the rocketfuel-get script.
206
134
download-cache:
167
download-cache:
207
168
ifdef LP_SOURCEDEPS_PATH
208
169
	utilities/link-external-sourcecode $(LP_SOURCEDEPS_PATH)/..
209
170
else
210
135
	@echo "Missing ./download-cache."
171
	@echo "Missing ./download-cache."
211
136
	@echo "Developers: please run utilities/link-external-sourcecode."
172
	@echo "Developers: please run utilities/link-external-sourcecode."
212
137
	@exit 1
173
	@exit 1
213
174
endif
214
175
215
176
buildonce_eggs: $(PY)
216
177
	find eggs -name '*.pyc' -exec rm {} \;
217
138
178
218
139
# The download-cache dependency comes *before* eggs so that developers get the
179
# The download-cache dependency comes *before* eggs so that developers get the
219
140
# warning before the eggs directory is made.  The target for the eggs directory
180
# warning before the eggs directory is made.  The target for the eggs directory
220
141
# is only there for deployment convenience.
181
# is only there for deployment convenience.
221
142
bin/buildout: download-cache eggs
182
bin/buildout: download-cache eggs
222
143
	$(SHHH) PYTHONPATH= $(PYTHON) bootstrap.py\
183
	$(SHHH) PYTHONPATH= $(PYTHON) bootstrap.py\
224
144
                --ez_setup-source=ez_setup.py \
184
		--setup-source=ez_setup.py \
225
145
		--download-base=download-cache/dist --eggs=eggs
185
		--download-base=download-cache/dist --eggs=eggs
226
146
186
228
147
$(PY): bin/buildout versions.cfg $(BUILDOUT_CFG) setup.py
187
# This builds bin/py and all the other bin files except bin/buildout.
229
188
# Remove the target before calling buildout to ensure that buildout
230
189
# updates the timestamp.
231
190
$(BUILDOUT_BIN): bin/buildout versions.cfg $(BUILDOUT_CFG) setup.py
232
191
	$(RM) $@
233
148
	$(SHHH) PYTHONPATH= ./bin/buildout \
192
	$(SHHH) PYTHONPATH= ./bin/buildout \
234
149
                configuration:instance_name=${LPCONFIG} -c $(BUILDOUT_CFG)
193
                configuration:instance_name=${LPCONFIG} -c $(BUILDOUT_CFG)
235
150
194
237
151
compile: $(PY)
195
compile: $(PY) $(BZR_VERSION_INFO)
238
152
	${SHHH} $(MAKE) -C sourcecode build PYTHON=${PYTHON} \
196
	${SHHH} $(MAKE) -C sourcecode build PYTHON=${PYTHON} \
239
153
	    PYTHON_VERSION=${PYTHON_VERSION} LPCONFIG=${LPCONFIG}
197
	    PYTHON_VERSION=${PYTHON_VERSION} LPCONFIG=${LPCONFIG}
244
154
	${SHHH} LPCONFIG=${LPCONFIG} $(PY) -t buildmailman.py
198
	${SHHH} LPCONFIG=${LPCONFIG} ${PY} -t buildmailman.py
241
155
	${SHHH} $(PY) sourcecode/lazr-js/tools/build.py \
242
156
		-n launchpad -s lib/canonical/launchpad/javascript \
243
157
		-b lib/canonical/launchpad/icing/build $(EXTRA_JS_FILES)
245
158
199
246
159
test_build: build
200
test_build: build
247
160
	bin/test $(TESTFLAGS) $(TESTOPTS)
201
	bin/test $(TESTFLAGS) $(TESTOPTS)
248
@@ -168,28 +209,38 @@
249
168
ftest_inplace: inplace
209
ftest_inplace: inplace
250
169
	bin/test -f $(TESTFLAGS) $(TESTOPTS)
210
	bin/test -f $(TESTFLAGS) $(TESTOPTS)
251
170
211
253
171
run: inplace stop
212
merge-proposal-jobs:
254
213
	# Handle merge proposal email jobs.
255
214
	$(PY) cronscripts/merge-proposal-jobs.py -v
256
215
257
216
run: check_schema inplace stop
258
172
	$(RM) thread*.request
217
	$(RM) thread*.request
260
173
	bin/run -r librarian,google-webservice -i $(LPCONFIG)
218
	bin/run -r librarian,google-webservice,memcached -i $(LPCONFIG)
261
174
219
263
175
start-gdb: inplace stop support_files
220
start-gdb: check_schema inplace stop support_files
264
176
	$(RM) thread*.request
221
	$(RM) thread*.request
265
177
	nohup gdb -x run.gdb --args bin/run -i $(LPCONFIG) \
222
	nohup gdb -x run.gdb --args bin/run -i $(LPCONFIG) \
266
178
		-r librarian,google-webservice
223
		-r librarian,google-webservice
267
179
		> ${LPCONFIG}-nohup.out 2>&1 &
224
		> ${LPCONFIG}-nohup.out 2>&1 &
268
180
225
270
181
run_all: inplace stop hosted_branches
226
run_all: check_schema inplace stop hosted_branches
271
182
	$(RM) thread*.request
227
	$(RM) thread*.request
273
183
	bin/run -r librarian,buildsequencer,sftp,mailman,codebrowse,google-webservice -i $(LPCONFIG)
228
	bin/run -r librarian,sftp,mailman,codebrowse,google-webservice,memcached \
274
229
	    -i $(LPCONFIG)
275
184
230
276
185
run_codebrowse: build
231
run_codebrowse: build
278
186
	BZR_PLUGIN_PATH=bzrplugins $(PY) sourcecode/launchpad-loggerhead/start-loggerhead.py -f
232
	BZR_PLUGIN_PATH=bzrplugins $(PY) scripts/start-loggerhead.py -f
279
187
233
280
188
start_codebrowse: build
234
start_codebrowse: build
282
189
	BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) sourcecode/launchpad-loggerhead/start-loggerhead.py
235
	BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) scripts/start-loggerhead.py
283
190
236
284
191
stop_codebrowse:
237
stop_codebrowse:
286
192
	$(PY) sourcecode/launchpad-loggerhead/stop-loggerhead.py
238
	$(PY) scripts/stop-loggerhead.py
287
239
288
240
run_codehosting: check_schema inplace stop hosted_branches
289
241
	$(RM) thread*.request
290
242
	bin/run -r librarian,sftp,codebrowse -i $(LPCONFIG)
291
243
292
193
244
293
194
start_librarian: build
245
start_librarian: build
294
195
	bin/start_librarian
246
	bin/start_librarian
295
@@ -202,14 +253,14 @@
296
202
253
297
203
scan_branches:
254
scan_branches:
298
204
	# Scan branches from the filesystem into the database.
255
	# Scan branches from the filesystem into the database.
300
205
	$(PY) cronscripts/branch-scanner.py
256
	$(PY) cronscripts/scan_branches.py
301
206
257
303
207
sync_branches: pull_branches scan_branches
258
sync_branches: pull_branches scan_branches merge-proposal-jobs
304
208
259
305
209
$(BZR_VERSION_INFO):
260
$(BZR_VERSION_INFO):
306
210
	scripts/update-bzr-version-info.sh
261
	scripts/update-bzr-version-info.sh
307
211
262
309
212
support_files: $(WADL_FILE) $(BZR_VERSION_INFO)
263
support_files: $(API_INDEX) $(BZR_VERSION_INFO)
310
213
264
311
214
# Intended for use on developer machines
265
# Intended for use on developer machines
312
215
start: inplace stop support_files initscript-start
266
start: inplace stop support_files initscript-start
313
@@ -231,7 +282,7 @@
314
231
# servers, where we know we don't need the extra steps in a full
282
# servers, where we know we don't need the extra steps in a full
315
232
# "make stop" because of how the code is deployed/built.
283
# "make stop" because of how the code is deployed/built.
316
233
initscript-stop:
284
initscript-stop:
318
234
	bin/killservice librarian buildsequencer launchpad mailman
285
	bin/killservice librarian launchpad mailman
319
235
286
320
236
shutdown: scheduleoutage stop
287
shutdown: scheduleoutage stop
321
237
	$(RM) +maintenancetime.txt
288
	$(RM) +maintenancetime.txt
322
@@ -242,18 +293,26 @@
323
242
	echo Sleeping ${MINS_TO_SHUTDOWN} mins
293
	echo Sleeping ${MINS_TO_SHUTDOWN} mins
324
243
	sleep ${MINS_TO_SHUTDOWN}m
294
	sleep ${MINS_TO_SHUTDOWN}m
325
244
295
327
245
harness:
296
harness: bin/harness
328
246
	bin/harness
297
	bin/harness
329
247
298
331
248
iharness:
299
iharness: bin/iharness
332
249
	bin/iharness
300
	bin/iharness
333
250
301
334
251
rebuildfti:
302
rebuildfti:
335
252
	@echo Rebuilding FTI indexes on launchpad_dev database
303
	@echo Rebuilding FTI indexes on launchpad_dev database
336
253
	$(PY) database/schema/fti.py -d launchpad_dev --force
304
	$(PY) database/schema/fti.py -d launchpad_dev --force
337
254
305
339
255
clean:
306
clean_js:
340
307
	$(RM) $(LP_BUILT_JS_ROOT)/launchpad.js
341
308
	$(RM) -r $(LAZR_BUILT_JS_ROOT)
342
309
343
310
clean: clean_js
344
256
	$(MAKE) -C sourcecode/pygettextpo clean
311
	$(MAKE) -C sourcecode/pygettextpo clean
345
312
	# XXX gary 2009-11-16 bug 483782
346
313
	# The pygettextpo Makefile should have this next line in it for its make
347
314
	# clean, and then we should remove this line.
348
315
	$(RM) sourcecode/pygpgme/gpgme/*.so
349
257
	if test -f sourcecode/mailman/Makefile; then \
316
	if test -f sourcecode/mailman/Makefile; then \
350
258
		$(MAKE) -C sourcecode/mailman clean; \
317
		$(MAKE) -C sourcecode/mailman clean; \
351
259
	fi
318
	fi
352
@@ -263,22 +322,24 @@
353
263
	    -print0 | xargs -r0 $(RM)
322
	    -print0 | xargs -r0 $(RM)
354
264
	$(RM) -r bin
323
	$(RM) -r bin
355
265
	$(RM) -r parts
324
	$(RM) -r parts
356
325
	$(RM) -r develop-eggs
357
266
	$(RM) .installed.cfg
326
	$(RM) .installed.cfg
358
267
	$(RM) -r build
327
	$(RM) -r build
359
268
	$(RM) thread*.request
328
	$(RM) thread*.request
360
269
	$(RM) -r lib/mailman
329
	$(RM) -r lib/mailman
361
270
	$(RM) -rf lib/canonical/launchpad/icing/build/*
330
	$(RM) -rf lib/canonical/launchpad/icing/build/*
362
271
	$(RM) -r $(CODEHOSTING_ROOT)
331
	$(RM) -r $(CODEHOSTING_ROOT)
364
272
	$(RM) $(WADL_FILE) $(API_INDEX)
332
	$(RM) $(APIDOC_DIR)/wadl*.xml $(APIDOC_DIR)/*.html
365
333
	$(RM) -rf $(APIDOC_DIR).tmp
366
273
	$(RM) $(BZR_VERSION_INFO)
334
	$(RM) $(BZR_VERSION_INFO)
367
274
	$(RM) _pythonpath.py
335
	$(RM) _pythonpath.py
368
336
	$(RM) +config-overrides.zcml
369
275
	$(RM) -rf \
337
	$(RM) -rf \
370
276
			  /var/tmp/builddmaster \
338
			  /var/tmp/builddmaster \
371
277
			  /var/tmp/bzrsync \
339
			  /var/tmp/bzrsync \
372
278
			  /var/tmp/codehosting.test \
340
			  /var/tmp/codehosting.test \
373
279
			  /var/tmp/codeimport \
341
			  /var/tmp/codeimport \
374
280
			  /var/tmp/fatsam.appserver \
342
			  /var/tmp/fatsam.appserver \
375
281
			  /var/tmp/launchpad_mailqueue \
376
282
			  /var/tmp/lperr \
343
			  /var/tmp/lperr \
377
283
			  /var/tmp/lperr.test \
344
			  /var/tmp/lperr.test \
378
284
			  /var/tmp/mailman \
345
			  /var/tmp/mailman \
379
@@ -286,6 +347,11 @@
380
286
			  /var/tmp/ppa \
347
			  /var/tmp/ppa \
381
287
			  /var/tmp/ppa.test \
348
			  /var/tmp/ppa.test \
382
288
			  /var/tmp/zeca
349
			  /var/tmp/zeca
383
350
	# /var/tmp/launchpad_mailqueue is created read-only on ec2test
384
351
	# instances.
385
352
	if [ -w /var/tmp/launchpad_mailqueue ]; then $(RM) -rf /var/tmp/launchpad_mailqueue; fi
386
353
	$(RM) -f lp.sfood lp-clustered.sfood lp-clustered.dot lp-clustered.svg
387
354
388
289
355
389
290
realclean: clean
356
realclean: clean
390
291
	$(RM) TAGS tags
357
	$(RM) TAGS tags
391
@@ -310,6 +376,8 @@
392
310
launchpad.pot:
376
launchpad.pot:
393
311
	bin/i18nextract.py
377
	bin/i18nextract.py
394
312
378
395
379
# Called by the rocketfuel-setup script. You probably don't want to run this
396
380
# on its own.
397
313
install: reload-apache
381
install: reload-apache
398
314
382
399
315
copy-certificates:
383
copy-certificates:
400
@@ -328,10 +396,7 @@
401
328
	a2ensite local-launchpad
396
	a2ensite local-launchpad
402
329
397
403
330
reload-apache: enable-apache-launchpad
398
reload-apache: enable-apache-launchpad
408
331
	/etc/init.d/apache2 reload
399
	/etc/init.d/apache2 restart
405
332
406
333
static:
407
334
	$(PY) scripts/make-static.py
409
335
400
410
336
TAGS: compile
401
TAGS: compile
411
337
	# emacs tags
402
	# emacs tags
412
@@ -345,8 +410,37 @@
413
345
	# idutils ID file
410
	# idutils ID file
414
346
	bin/tags -i
411
	bin/tags -i
415
347
412
416
413
lp.sfood:
417
414
	# Generate import dependency graph
418
415
	sfood -i -u -I lib/sqlobject -I lib/schoolbell -I lib/devscripts -I lib/contrib \
419
416
	-I lib/canonical/not-used lib/canonical lib/lp 2>/dev/null | grep -v contrib/ \
420
417
	| grep -v sqlobject | grep -v BeautifulSoup | grep -v psycopg \
421
418
	| grep -v schoolbell > lp.sfood.tmp
422
419
	mv lp.sfood.tmp lp.sfood
423
420
424
421
425
422
lp-clustered.sfood: lp.sfood lp-sfood-packages
426
423
	# Cluster the import dependency graph
427
424
	sfood-cluster -f lp-sfood-packages < lp.sfood > lp-clustered.sfood.tmp
428
425
	mv lp-clustered.sfood.tmp lp-clustered.sfood
429
426
430
427
431
428
lp-clustered.dot: lp-clustered.sfood
432
429
	# Build the visual graph
433
430
	sfood-graph -p < lp-clustered.sfood > lp-clustered.dot.tmp
434
431
	mv lp-clustered.dot.tmp lp-clustered.dot
435
432
436
433
437
434
lp-clustered.svg: lp-clustered.dot
438
435
	# Render to svg
439
436
	dot -Tsvg < lp-clustered.dot > lp-clustered.svg.tmp
440
437
	mv lp-clustered.svg.tmp lp-clustered.svg
441
438
442
439
443
348
.PHONY: apidoc check tags TAGS zcmldocs realclean clean debug stop\
440
.PHONY: apidoc check tags TAGS zcmldocs realclean clean debug stop\
444
349
	start run ftest_build ftest_inplace test_build test_inplace pagetests\
441
	start run ftest_build ftest_inplace test_build test_inplace pagetests\
446
350
	check check_loggerhead_on_merge  check_merge check_sourcecode_merge \
442
	check check_merge \
447
351
	schema default launchpad.pot check_merge_ui pull scan sync_branches\
443
	schema default launchpad.pot check_merge_ui pull scan sync_branches\
449
352
	reload-apache hosted_branches check_db_merge check_mailman check_config
444
	reload-apache hosted_branches check_db_merge check_mailman check_config\
450
445
	jsbuild jsbuild_lazr clean_js buildonce_eggs \
451
446
	sprite_css sprite_image css_combine compile check_schema
452
353
447
453
=== modified file 'bootstrap.py'
454
--- bootstrap.py	2009-08-05 18:52:52 +0000
455
+++ bootstrap.py	2010-05-05 01:58:48 +0000
456
@@ -17,154 +17,190 @@
457
17
The script accepts buildout command-line options, so you can
17
The script accepts buildout command-line options, so you can
458
18
use the -c option to specify an alternate configuration file.
18
use the -c option to specify an alternate configuration file.
459
19
19
461
20
$Id: bootstrap.py 101930 2009-07-15 18:34:35Z gary $
20
$Id$
462
21
"""
21
"""
463
22
22
524
23
import os, re, shutil, sys, tempfile, textwrap, urllib, urllib2
23
import os, shutil, sys, tempfile, textwrap, urllib, urllib2
525
24
24
from optparse import OptionParser
526
25
# We have to manually parse our options rather than using one of the stdlib
25
527
26
# tools because we want to pass the ones we don't recognize along to
26
if sys.platform == 'win32':
528
27
# zc.buildout.buildout.main.
27
    def quote(c):
529
28
28
        if ' ' in c:
530
29
configuration = {
29
            return '"%s"' % c # work around spawn lamosity on windows
471
30
    '--ez_setup-source': 'http://peak.telecommunity.com/dist/ez_setup.py',
472
31
    '--version': '',
473
32
    '--download-base': None,
474
33
    '--eggs': None}
475
34
476
35
helpstring = __doc__ + textwrap.dedent('''
477
36
    This script recognizes the following options itself.  The first option it
478
37
    encounters that is not one of these will cause the script to stop parsing
479
38
    options and pass the rest on to buildout.  Therefore, if you want to use
480
39
    any of the following options *and* buildout command-line options like
481
40
    -c, first use the following options, and then use the buildout options.
482
41
483
42
    Options: 
484
43
      --version=ZC_BUILDOUT_VERSION
485
44
                Specify a version number of the zc.buildout to use
486
45
      --ez_setup-source=URL_OR_FILE
487
46
                Specify a URL or file location for the ez_setup file.
488
47
                Defaults to
489
48
                %(--ez_setup-source)s
490
49
      --download-base=URL_OR_DIRECTORY
491
50
                Specify a URL or directory for downloading setuptools and
492
51
                zc.buildout.  Defaults to PyPI.
493
52
      --eggs=DIRECTORY
494
53
                Specify a directory for storing eggs.  Defaults to a temporary
495
54
                directory that is deleted when the bootstrap script completes.
496
55
497
56
    By using --ez_setup-source and --download-base to point to local resources,
498
57
    you can keep this script from going over the network.
499
58
    ''' % configuration)
500
59
match_equals = re.compile(r'(%s)=(.*)' % ('|'.join(configuration),)).match
501
60
args = sys.argv[1:]
502
61
if args == ['--help']:
503
62
    print helpstring
504
63
    sys.exit(0)
505
64
506
65
# If we end up using a temporary directory for storing our eggs, this will
507
66
# hold the path of that directory.  On the other hand, if an explicit directory
508
67
# is specified in the argv, this will remain None.
509
68
tmpeggs = None
510
69
511
70
while args:
512
71
    val = args[0]
513
72
    if val in configuration:
514
73
        del args[0]
515
74
        if not args or args[0].startswith('-'):
516
75
            print "ERROR: %s requires an argument."
517
76
            print helpstring
518
77
            sys.exit(1)
519
78
        configuration[val] = args[0]
520
79
    else:
521
80
        match = match_equals(val)
522
81
        if match and match.group(1) in configuration:
523
82
            configuration[match.group(1)] = match.group(2)
531
83
        else:
30
        else:
556
84
            break
31
            return c
557
85
    del args[0]
32
else:
558
86
33
    quote = str
559
87
for name in ('--ez_setup-source', '--download-base'):
34
560
88
    val = configuration[name]
35
# In order to be more robust in the face of system Pythons, we want to
561
89
    if val is not None and '://' not in val: # We're being lazy.
36
# run without site-packages loaded.  This is somewhat tricky, in
562
90
        configuration[name] = 'file://%s' % (
37
# particular because Python 2.6's distutils imports site, so starting
563
91
            urllib.pathname2url(os.path.abspath(os.path.expanduser(val))),)
38
# with the -S flag is not sufficient.  However, we'll start with that:
564
92
39
if 'site' in sys.modules:
565
93
if (configuration['--download-base'] and
40
    # We will restart with python -S.
566
94
    not configuration['--download-base'].endswith('/')):
41
    args = sys.argv[:]
567
95
    # Download base needs a trailing slash to make the world happy.
42
    args[0:0] = [sys.executable, '-S']
568
96
    configuration['--download-base'] += '/'
43
    args = map(quote, args)
569
97
44
    os.execv(sys.executable, args)
570
98
if not configuration['--eggs']:
45
# Now we are running with -S.  We'll get the clean sys.path, import site
571
99
    configuration['--eggs'] = tmpeggs = tempfile.mkdtemp()
46
# because distutils will do it later, and then reset the path and clean
572
100
else:
47
# out any namespace packages from site-packages that might have been
573
101
    configuration['--eggs'] = os.path.abspath(
48
# loaded by .pth files.
574
102
        os.path.expanduser(configuration['--eggs']))
49
clean_path = sys.path[:]
575
103
50
import site
576
104
# The requirement is what we will pass to setuptools to specify zc.buildout.
51
sys.path[:] = clean_path
577
105
requirement = 'zc.buildout'
52
for k, v in sys.modules.items():
578
106
if configuration['--version']:
53
    if (hasattr(v, '__path__') and
579
107
    requirement += '==' + configuration['--version']
54
        len(v.__path__)==1 and
580
55
        not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))):
581
56
        # This is a namespace package.  Remove it.
582
57
        sys.modules.pop(k)
583
58
584
59
is_jython = sys.platform.startswith('java')
585
60
586
61
setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
587
62
distribute_source = 'http://python-distribute.org/distribute_setup.py'
588
63
589
64
# parsing arguments
590
65
def normalize_to_url(option, opt_str, value, parser):
591
66
    if value:
592
67
        if '://' not in value: # It doesn't smell like a URL.
593
68
            value = 'file://%s' % (
594
69
                urllib.pathname2url(
595
70
                    os.path.abspath(os.path.expanduser(value))),)
596
71
        if opt_str == '--download-base' and not value.endswith('/'):
597
72
            # Download base needs a trailing slash to make the world happy.
598
73
            value += '/'
599
74
    else:
600
75
        value = None
601
76
    name = opt_str[2:].replace('-', '_')
602
77
    setattr(parser.values, name, value)
603
78
604
79
usage = '''\
605
80
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
606
81
607
82
Bootstraps a buildout-based project.
608
83
609
84
Simply run this script in a directory containing a buildout.cfg, using the
610
85
Python that you want bin/buildout to use.
611
86
612
87
Note that by using --setup-source and --download-base to point to
613
88
local resources, you can keep this script from going over the network.
614
89
'''
615
90
616
91
parser = OptionParser(usage=usage)
617
92
parser.add_option("-v", "--version", dest="version",
618
93
                          help="use a specific zc.buildout version")
619
94
parser.add_option("-d", "--distribute",
620
95
                   action="store_true", dest="use_distribute", default=False,
621
96
                   help="Use Distribute rather than Setuptools.")
622
97
parser.add_option("--setup-source", action="callback", dest="setup_source",
623
98
                  callback=normalize_to_url, nargs=1, type="string",
624
99
                  help=("Specify a URL or file location for the setup file. "
625
100
                        "If you use Setuptools, this will default to " +
626
101
                        setuptools_source + "; if you use Distribute, this "
627
102
                        "will default to " + distribute_source +"."))
628
103
parser.add_option("--download-base", action="callback", dest="download_base",
629
104
                  callback=normalize_to_url, nargs=1, type="string",
630
105
                  help=("Specify a URL or directory for downloading "
631
106
                        "zc.buildout and either Setuptools or Distribute. "
632
107
                        "Defaults to PyPI."))
633
108
parser.add_option("--eggs",
634
109
                  help=("Specify a directory for storing eggs.  Defaults to "
635
110
                        "a temporary directory that is deleted when the "
636
111
                        "bootstrap script completes."))
637
112
parser.add_option("-c", None, action="store", dest="config_file",
638
113
                   help=("Specify the path to the buildout configuration "
639
114
                         "file to be used."))
640
115
641
116
options, args = parser.parse_args()
642
117
643
118
# if -c was provided, we push it back into args for buildout' main function
644
119
if options.config_file is not None:
645
120
    args += ['-c', options.config_file]
646
121
647
122
if options.eggs:
648
123
    eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))
649
124
else:
650
125
    eggs_dir = tempfile.mkdtemp()
651
126
652
127
if options.setup_source is None:
653
128
    if options.use_distribute:
654
129
        options.setup_source = distribute_source
655
130
    else:
656
131
        options.setup_source = setuptools_source
657
132
658
133
args = args + ['bootstrap']
659
134
660
108
135
661
109
try:
136
try:
662
137
    to_reload = False
663
138
    import pkg_resources
664
139
    to_reload = True
665
140
    if not hasattr(pkg_resources, '_distribute'):
666
141
        raise ImportError
667
110
    import setuptools # A flag.  Sometimes pkg_resources is installed alone.
142
    import setuptools # A flag.  Sometimes pkg_resources is installed alone.
668
111
    import pkg_resources
669
112
except ImportError:
143
except ImportError:
670
144
    ez_code = urllib2.urlopen(
671
145
        options.setup_source).read().replace('\r\n', '\n')
672
113
    ez = {}
146
    ez = {}
679
114
    exec urllib2.urlopen(configuration['--ez_setup-source']).read() in ez
147
    exec ez_code in ez
680
115
    setuptools_args = dict(to_dir=configuration['--eggs'], download_delay=0)
148
    setup_args = dict(to_dir=eggs_dir, download_delay=0)
681
116
    if configuration['--download-base']:
149
    if options.download_base:
682
117
        setuptools_args['download_base'] = configuration['--download-base']
150
        setup_args['download_base'] = options.download_base
683
118
    ez['use_setuptools'](**setuptools_args)
151
    if options.use_distribute:
684
119
    import pkg_resources
152
        setup_args['no_fake'] = True
685
153
    ez['use_setuptools'](**setup_args)
686
154
    if to_reload:
687
155
        reload(pkg_resources)
688
156
    else:
689
157
        import pkg_resources
690
120
    # This does not (always?) update the default working set.  We will
158
    # This does not (always?) update the default working set.  We will
691
121
    # do it.
159
    # do it.
692
122
    for path in sys.path:
160
    for path in sys.path:
693
123
        if path not in pkg_resources.working_set.entries:
161
        if path not in pkg_resources.working_set.entries:
694
124
            pkg_resources.working_set.add_entry(path)
162
            pkg_resources.working_set.add_entry(path)
695
125
163
696
126
if sys.platform == 'win32':
697
127
    def quote(c):
698
128
        if ' ' in c:
699
129
            return '"%s"' % c # work around spawn lamosity on windows
700
130
        else:
701
131
            return c
702
132
else:
703
133
    def quote (c):
704
134
        return c
705
135
cmd = [quote(sys.executable),
164
cmd = [quote(sys.executable),
706
136
       '-c',
165
       '-c',
707
137
       quote('from setuptools.command.easy_install import main; main()'),
166
       quote('from setuptools.command.easy_install import main; main()'),
708
138
       '-mqNxd',
167
       '-mqNxd',
714
139
       quote(configuration['--eggs'])]
168
       quote(eggs_dir)]
715
140
169
716
141
if configuration['--download-base']:
170
if options.download_base:
717
142
    cmd.extend(['-f', quote(configuration['--download-base'])])
171
    cmd.extend(['-f', quote(options.download_base)])
718
143
172
719
173
requirement = 'zc.buildout'
720
174
if options.version:
721
175
    requirement = '=='.join((requirement, options.version))
722
144
cmd.append(requirement)
176
cmd.append(requirement)
723
145
177
724
178
if options.use_distribute:
725
179
    setup_requirement = 'distribute'
726
180
else:
727
181
    setup_requirement = 'setuptools'
728
146
ws = pkg_resources.working_set
182
ws = pkg_resources.working_set
729
147
env = dict(
183
env = dict(
730
148
    os.environ,
184
    os.environ,
732
149
    PYTHONPATH=ws.find(pkg_resources.Requirement.parse('setuptools')).location)
185
    PYTHONPATH=ws.find(
733
186
        pkg_resources.Requirement.parse(setup_requirement)).location)
734
150
187
735
151
is_jython = sys.platform.startswith('java')
736
152
if is_jython:
188
if is_jython:
737
153
    import subprocess
189
    import subprocess
738
154
    exitcode = subprocess.Popen(cmd, env=env).wait()
190
    exitcode = subprocess.Popen(cmd, env=env).wait()
740
155
else: # Windows needs this, apparently; otherwise we would prefer subprocess
191
else: # Windows prefers this, apparently; otherwise we would prefer subprocess
741
156
    exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
192
    exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
742
157
if exitcode != 0:
193
if exitcode != 0:
743
158
    sys.stdout.flush()
194
    sys.stdout.flush()
744
195
    sys.stderr.flush()
745
159
    print ("An error occured when trying to install zc.buildout. "
196
    print ("An error occured when trying to install zc.buildout. "
746
160
           "Look above this message for any errors that "
197
           "Look above this message for any errors that "
747
161
           "were output by easy_install.")
198
           "were output by easy_install.")
748
162
    sys.exit(exitcode)
199
    sys.exit(exitcode)
749
163
200
751
164
ws.add_entry(configuration['--eggs'])
201
ws.add_entry(eggs_dir)
752
165
ws.require(requirement)
202
ws.require(requirement)
753
166
import zc.buildout.buildout
203
import zc.buildout.buildout
754
167
args.append('bootstrap')
755
168
zc.buildout.buildout.main(args)
204
zc.buildout.buildout.main(args)
758
169
if tmpeggs is not None:
205
if not options.eggs: # clean up temporary egg directory
759
170
    shutil.rmtree(tmpeggs)
206
    shutil.rmtree(eggs_dir)
760
171
207
761
=== modified file 'buildmailman.py'
762
--- buildmailman.py	2009-09-11 02:17:29 +0000
763
+++ buildmailman.py	2010-05-05 01:58:48 +0000
764
@@ -1,4 +1,4 @@
766
1
#! /usr/bin/python2.4
1
#! /usr/bin/python2.5
767
2
#
2
#
768
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
769
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
770
@@ -84,25 +84,52 @@
771
84
84
772
85
    # Build and install the Mailman software.  Note that we don't care about
85
    # Build and install the Mailman software.  Note that we don't care about
773
86
    # --with-cgi-gid because we're not going to use that Mailman subsystem.
86
    # --with-cgi-gid because we're not going to use that Mailman subsystem.
774
87
    executable = os.path.abspath('bin/py')
775
87
    configure_args = (
88
    configure_args = (
776
88
        './configure',
89
        './configure',
777
89
        '--prefix', mailman_path,
90
        '--prefix', mailman_path,
778
90
        '--with-var-prefix=' + var_dir,
91
        '--with-var-prefix=' + var_dir,
780
91
        '--with-python=' + sys.executable,
92
        '--with-python=' + executable,
781
92
        '--with-username=' + user,
93
        '--with-username=' + user,
782
93
        '--with-groupname=' + group,
94
        '--with-groupname=' + group,
783
94
        '--with-mail-gid=' + group,
95
        '--with-mail-gid=' + group,
784
95
        '--with-mailhost=' + build_host_name,
96
        '--with-mailhost=' + build_host_name,
785
96
        '--with-urlhost=' + build_host_name,
97
        '--with-urlhost=' + build_host_name,
786
97
        )
98
        )
787
99
    # Configure.
788
98
    retcode = subprocess.call(configure_args, cwd=mailman_source)
100
    retcode = subprocess.call(configure_args, cwd=mailman_source)
789
99
    if retcode:
101
    if retcode:
790
100
        print >> sys.stderr, 'Could not configure Mailman:'
102
        print >> sys.stderr, 'Could not configure Mailman:'
791
101
        sys.exit(retcode)
103
        sys.exit(retcode)
792
104
    # Make.
793
102
    retcode = subprocess.call(('make',), cwd=mailman_source)
105
    retcode = subprocess.call(('make',), cwd=mailman_source)
794
103
    if retcode:
106
    if retcode:
795
104
        print >> sys.stderr, 'Could not make Mailman.'
107
        print >> sys.stderr, 'Could not make Mailman.'
796
105
        sys.exit(retcode)
108
        sys.exit(retcode)
797
109
    # We have a brief interlude before we install.  Hardy will not
798
110
    # accept a script as the executable for the shebang line--it will
799
111
    # treat the file as a shell script instead. The ``bin/by``
800
112
    # executable that we specified in '--with-python' above is a script
801
113
    # so this behavior causes problems for us. Our work around is to
802
114
    # prefix the ``bin/py`` script with ``/usr/bin/env``, which makes
803
115
    # Hardy happy.  We need to do this before we install because the
804
116
    # installation will call Mailman's ``bin/update``, which is a script
805
117
    # that needs this fix.
806
118
    build_dir = os.path.join(mailman_source, 'build')
807
119
    original = '#! %s\n' % (executable,)
808
120
    modified = '#! /usr/bin/env %s\n' % (executable,)
809
121
    for (dirpath, dirnames, filenames) in os.walk(build_dir):
810
122
        for filename in filenames:
811
123
            filename = os.path.join(dirpath, filename)
812
124
            f = open(filename, 'r')
813
125
            if f.readline() == original:
814
126
                rest = f.read()
815
127
                f.close()
816
128
                f = open(filename, 'w')
817
129
                f.write(modified)
818
130
                f.write(rest)
819
131
            f.close()
820
132
    # Now we actually install.
821
106
    retcode = subprocess.call(('make', 'install'), cwd=mailman_source)
133
    retcode = subprocess.call(('make', 'install'), cwd=mailman_source)
822
107
    if retcode:
134
    if retcode:
823
108
        print >> sys.stderr, 'Could not install Mailman.'
135
        print >> sys.stderr, 'Could not install Mailman.'
824
109
136
825
=== modified file 'buildout-templates/_pythonpath.py.in'
826
--- buildout-templates/_pythonpath.py.in	2009-08-21 19:13:05 +0000
827
+++ buildout-templates/_pythonpath.py.in	2010-05-05 01:58:48 +0000
828
@@ -4,17 +4,36 @@
829
4
# NOTE: This is a generated file.  The original is in
4
# NOTE: This is a generated file.  The original is in
830
5
# buildout-templates/_pythonpath.py.in
5
# buildout-templates/_pythonpath.py.in
831
6
6
841
7
__metaclass__ = type
7
# This file works if the Python has been started with -S, or if bin/py
842
8
8
# has been used.
843
9
import sys, os
9
844
10
10
# Auto-generated code to handle relative paths
845
11
sys.path[0:0] = [${string-paths}]
11
${python-relative-path-setup}
846
12
# Enable Storm's C extensions
12
847
13
os.environ['STORM_CEXTENSIONS'] = '1'
13
import os
848
14
14
import sys
840
15
# We don't want to bother tests or logs with these.
849
16
import warnings
15
import warnings
850
16
851
17
# XXX: 2010-04-26, Salgado, bug=570246: Silence python2.6 deprecation
852
18
# warnings.
853
17
warnings.filterwarnings(
19
warnings.filterwarnings(
857
18
    'ignore',
20
    'ignore', '.*(md5|sha|sets)', DeprecationWarning,
858
19
    'Module .+ was already imported from .+, but .+ is being added.*',
21
    )
859
20
    UserWarning)
22
860
23
site_dir = ${scripts:parts-directory|path-repr}
861
24
862
25
if ('site' in sys.modules and
863
26
    not sys.modules['site'].__file__.startswith(
864
27
        os.path.join(site_dir, 'site.py'))):
865
28
    # We have the wrong site.py, so our paths are not set up correctly.
866
29
    # We blow up, with a hopefully helpful error message.
867
30
    raise RuntimeError(
868
31
        'The wrong site.py is imported (%r imported, %r expected). '
869
32
        'Scripts should usually be '
870
33
        "started with Launchpad's bin/py, or with a Python invoked with "
871
34
        'the -S flag.' % (
872
35
        sys.modules['site'].__file__, os.path.join(site_dir, 'site.py')))
873
36
874
37
if site_dir not in sys.path:
875
38
    sys.path.insert(0, site_dir)
876
39
import site # sets up paths
877
21
40
878
=== added file 'buildout-templates/bin/combine-css.in'
879
--- buildout-templates/bin/combine-css.in	1970-01-01 00:00:00 +0000
880
+++ buildout-templates/bin/combine-css.in	2010-05-05 01:58:48 +0000
881
@@ -0,0 +1,51 @@
882
1
#!${buildout:executable} -S
883
2
884
3
# Initialize our paths.
885
4
${python-relative-path-setup}
886
5
import sys
887
6
sys.path.insert(0, ${scripts:parts-directory|path-repr})
888
7
import site
889
8
890
9
import os
891
10
892
11
from lazr.js.build import ComboFile
893
12
from lazr.js.combo import combine_files
894
13
895
14
896
15
root = ${buildout:directory|path-repr}
897
16
icing = os.path.join(root, 'lib/canonical/launchpad/icing')
898
17
target = os.path.join(icing, 'combo.css')
899
18
# It'd probably be nice to have this script find all the CSS files we might
900
19
# need and combine them together, but if we do that we'd certainly end up
901
20
# including lots of styles that we don't need/want, so keeping this hard-coded
902
21
# list seems like the best option for now.
903
22
names = [
904
23
    'style.css',
905
24
    'lazr/build/yui/cssreset/reset.css',
906
25
    'lazr/build/yui/cssfonts/fonts.css',
907
26
    'lazr/build/yui/cssgrids/grids.css',
908
27
    'lazr/build/lazr/assets/skins/sam/lazr.css',
909
28
    'lazr/build/inlineedit/assets/skins/sam/editor.css',
910
29
    'lazr/build/autocomplete/assets/skins/sam/autocomplete.css',
911
30
    'lazr/build/overlay/assets/skins/sam/pretty-overlay.css',
912
31
    'lazr/build/formoverlay/assets/formoverlay-core.css',
913
32
    'lazr/build/picker/assets/skins/sam/picker.css',
914
33
    'lazr/build/activator/assets/skins/sam/activator.css',
915
34
    'lazr/build/choiceedit/assets/choiceedit-core.css',
916
35
    # This one goes at the end because it's our main stylesheet and should
917
36
    # take precedence over the others.
918
37
    'build/style-3-0.css']
919
38
920
39
absolute_names = []
921
40
for name in names:
922
41
    absolute_names.append(os.path.join(icing, name))
923
42
924
43
combo = ComboFile(absolute_names, target)
925
44
if combo.needs_update():
926
45
    result = ''
927
46
    for content in combine_files(names, icing):
928
47
        result += content
929
48
930
49
    f = open(target, 'w')
931
50
    f.write(result)
932
51
    f.close()
933
0
52
934
=== modified file 'buildout-templates/bin/jstest.in'
935
--- buildout-templates/bin/jstest.in	2009-10-07 10:19:19 +0000
936
+++ buildout-templates/bin/jstest.in	2010-05-05 01:58:48 +0000
937
@@ -1,4 +1,4 @@
939
1
#!${buildout:executable}
1
#!${buildout:executable} -S
940
2
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2
# Copyright 2009 Canonical Ltd.  This software is licensed under the
941
3
# GNU Affero General Public License version 3 (see the file LICENSE).
3
# GNU Affero General Public License version 3 (see the file LICENSE).
942
4
4
943
@@ -14,10 +14,11 @@
944
14
14
945
15
"""
15
"""
946
16
16
948
17
# Add the buildout sys.path
17
# Initialize our paths.
949
18
${python-relative-path-setup}
950
18
import sys
19
import sys
953
19
sys.path[0:0] = [${string-paths}]
20
sys.path.insert(0, ${scripts:parts-directory|path-repr})
954
20
21
import site
955
21
22
956
22
import subprocess
23
import subprocess
957
23
import os
24
import os
958
24
25
959
=== added file 'buildout-templates/bin/kill-test-services.in'
960
--- buildout-templates/bin/kill-test-services.in	1970-01-01 00:00:00 +0000
961
+++ buildout-templates/bin/kill-test-services.in	2010-05-05 01:58:48 +0000
962
@@ -0,0 +1,42 @@
963
1
#!${buildout:executable} -S
964
2
#
965
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
966
4
# GNU Affero General Public License version 3 (see the file LICENSE).
967
5
"""Kill all the test services that may persist between test runs."""
968
6
969
7
# Initialize our paths.
970
8
${python-relative-path-setup}
971
9
import sys
972
10
sys.path.insert(0, ${scripts:parts-directory|path-repr})
973
11
import site
974
12
975
13
# Tell canonical.config to use the testrunner config instance, so that
976
14
# we don't kill the real services.
977
15
from canonical.config import config
978
16
config.setInstance('testrunner')
979
17
config.generate_overrides()
980
18
981
19
import sys
982
20
983
21
from canonical.testing.layers import MemcachedLayer
984
22
from canonical.librarian.ftests.harness import (
985
23
    LibrarianTestSetup, TacLibrarianTestSetup)
986
24
from lp.services.osutils import kill_by_pidfile
987
25
988
26
989
27
def main(args):
990
28
    if '-h' in args or '--help' in args:
991
29
        print __doc__
992
30
        return 0
993
31
    print "Killing Memcached....",
994
32
    kill_by_pidfile(MemcachedLayer.getPidFile())
995
33
    print "done."
996
34
    print "Killing Librarian....",
997
35
    TacLibrarianTestSetup().tearDown()
998
36
    LibrarianTestSetup().tearDownRoot()
999
37
    print "done."
1000
38
    return 0
1001
39
1002
40
1003
41
if __name__ == '__main__':
1004
42
    sys.exit(main(sys.argv[1:]))
1005
0
43
1006
=== modified file 'buildout-templates/bin/lint.sh.in'
1007
--- buildout-templates/bin/lint.sh.in	2009-07-17 00:26:05 +0000
1008
+++ buildout-templates/bin/lint.sh.in	2010-05-05 01:58:48 +0000
1009
@@ -6,7 +6,9 @@
1010
6
# Runs xmlint, pyflakes and pylint on files changed from parent branch.
6
# Runs xmlint, pyflakes and pylint on files changed from parent branch.
1011
7
# Use '-v' to run pylint under stricter conditions with additional messages.
7
# Use '-v' to run pylint under stricter conditions with additional messages.
1012
8
8
1014
9
utilitiesdir=`dirname $0`/../utilities
9
${shell-relative-path-setup}
1015
10
1016
11
utilitiesdir=${buildout:directory/utilities|shell-path}
1017
10
[ -z "$utilitiesdir" ] && utilitiesdir=.
12
[ -z "$utilitiesdir" ] && utilitiesdir=.
1018
11
13
1019
12
# Fail if any of the required tools are not installed.
14
# Fail if any of the required tools are not installed.
1020
@@ -56,9 +58,15 @@
1021
56
            # This is a loom, lint changes relative to the lower thread.
58
            # This is a loom, lint changes relative to the lower thread.
1022
57
            rev_option="-r thread:"
59
            rev_option="-r thread:"
1023
58
        else
60
        else
1027
59
            # Lint changes relative to the parent.
61
            if test "$(bzr pipes | sed -n -e "/^\\*/q;p" | wc -l)" -gt 0 ; then
1028
60
            rev=`bzr info | sed '/parent branch:/!d; s/ *parent branch: /ancestor:/'`
62
                # This is a pipeline with at least one pipe before the current,
1029
61
            rev_option="-r $rev"
63
                # lint changes relative to the previous pipe
1030
64
                rev_option="-r ancestor::prev"
1031
65
            else
1032
66
                # Lint changes relative to the parent.
1033
67
                rev=`bzr info | sed '/parent branch:/!d; s/ *parent branch: /ancestor:/'`
1034
68
                rev_option="-r $rev"
1035
69
            fi
1036
62
        fi
70
        fi
1037
63
    elif [ $diff_status -eq 1 ] ; then
71
    elif [ $diff_status -eq 1 ] ; then
1038
64
        # Uncommitted changes in the tree, lint those changes.
72
        # Uncommitted changes in the tree, lint those changes.
1039
@@ -147,7 +155,7 @@
1040
147
    echo "        cp $4 $1"
155
    echo "        cp $4 $1"
1041
148
    echo "    Run make schema again to update the test/dev database."
156
    echo "    Run make schema again to update the test/dev database."
1042
149
}
157
}
1044
150
    
158
1045
151
if [ -n "$sql_diff" -o -n "$sql_dev_diff" -o -n "$karma_bombs" ]; then
159
if [ -n "$sql_diff" -o -n "$sql_dev_diff" -o -n "$karma_bombs" ]; then
1046
152
    echo ""
160
    echo ""
1047
153
    echo ""
161
    echo ""
1048
@@ -155,7 +163,7 @@
1049
155
    echo ""
163
    echo ""
1050
156
fi
164
fi
1051
157
165
1053
158
# 
166
#
1054
159
if [ -n "$sql_diff" -o -n "$karma_bombs" ]; then
167
if [ -n "$sql_diff" -o -n "$karma_bombs" ]; then
1055
160
    echo "$current_sql"
168
    echo "$current_sql"
1056
161
fi
169
fi
1057
@@ -245,7 +253,7 @@
1058
245
253
1059
246
jsfiles=`echo "$files" | grep -E 'js$'`
254
jsfiles=`echo "$files" | grep -E 'js$'`
1060
247
if [ ! -z "$jsfiles" ]; then
255
if [ ! -z "$jsfiles" ]; then
1062
248
    jslint_notices=`$utilitiesdir/../sourcecode/lazr-js/tools/jslint.py 2>&1`
256
    jslint_notices=`$utilitiesdir/../bin/jslint 2>&1`
1063
249
    if [ ! -z "$jslint_notices" ]; then
257
    if [ ! -z "$jslint_notices" ]; then
1064
250
        echo ""
258
        echo ""
1065
251
        echo ""
259
        echo ""
1066
@@ -273,7 +281,7 @@
1067
273
fi
281
fi
1068
274
282
1069
275
extra_path="/usr/share/pyshared:/usr/share/pycentral/pylint/site-packages"
283
extra_path="/usr/share/pyshared:/usr/share/pycentral/pylint/site-packages"
1071
276
export PYTHONPATH="${os-paths}:$extra_path:$PYTHONPATH"
284
export PYTHONPATH=${scripts:parts-directory|shell-path}:"$extra_path":"$PYTHONPATH"
1072
277
pylint="${buildout:executable} -Wi::DeprecationWarning `which pylint`"
285
pylint="${buildout:executable} -Wi::DeprecationWarning `which pylint`"
1073
278
286
1074
279
# XXX sinzui 2007-10-18 bug=154140:
287
# XXX sinzui 2007-10-18 bug=154140:
1075
280
288
1076
=== removed file 'buildout-templates/bin/py.in'
1077
--- buildout-templates/bin/py.in	2009-06-12 16:36:02 +0000
1078
+++ buildout-templates/bin/py.in	1970-01-01 00:00:00 +0000
1079
@@ -1,2 +0,0 @@
1080
1
#!/bin/sh
1081
2
PYTHONPATH=${os-paths} exec ${buildout:executable} "$@"
1082
3
0
1083
=== modified file 'buildout-templates/bin/retest.in'
1084
--- buildout-templates/bin/retest.in	2009-10-01 10:12:28 +0000
1085
+++ buildout-templates/bin/retest.in	2010-05-05 01:58:48 +0000
1086
@@ -31,9 +31,10 @@
1087
31
from itertools import takewhile
31
from itertools import takewhile
1088
32
from pprint import pprint
32
from pprint import pprint
1089
33
33
1090
34
${python-relative-path-setup}
1091
34
35
1092
35
# The test script for this branch.
36
# The test script for this branch.
1094
36
TEST = '${buildout:directory}/bin/test'
37
TEST = ${buildout:directory/bin/test|path-repr}
1095
37
38
1096
38
# Regular expression to match numbered stories.
39
# Regular expression to match numbered stories.
1097
39
STORY_RE = re.compile("(.*)/\d{2}-.*")
40
STORY_RE = re.compile("(.*)/\d{2}-.*")
1098
40
41
1099
=== added file 'buildout-templates/bin/sprite-util.in'
1100
--- buildout-templates/bin/sprite-util.in	1970-01-01 00:00:00 +0000
1101
+++ buildout-templates/bin/sprite-util.in	2010-05-05 01:58:48 +0000
1102
@@ -0,0 +1,47 @@
1103
1
#!${buildout:executable} -S
1104
2
1105
3
import os
1106
4
import sys
1107
5
1108
6
# Initialize our paths.
1109
7
${python-relative-path-setup}
1110
8
sys.path.insert(0, ${scripts:parts-directory|path-repr})
1111
9
import site
1112
10
1113
11
from lp.services.spriteutils import SpriteUtil
1114
12
1115
13
command_options = ('create-image', 'create-css')
1116
14
1117
15
def usage():
1118
16
    return " Usage: %s %s" % (sys.argv[0], '|'.join(command_options))
1119
17
1120
18
if len(sys.argv) != 2:
1121
19
    print >> sys.stderr, "Expected a single argument."
1122
20
    print >> sys.stderr, usage()
1123
21
    sys.exit(1)
1124
22
else:
1125
23
    command = sys.argv[1]
1126
24
    if command not in command_options:
1127
25
        print >> sys.stderr, "Unknown argument: %s" % command
1128
26
        print >> sys.stderr, usage()
1129
27
        sys.exit(2)
1130
28
1131
29
icing = ${buildout:directory/lib/canonical/launchpad/icing|path-repr}
1132
30
combined_image_file = os.path.join(icing, 'icon-sprites')
1133
31
positioning_file = os.path.join(icing, 'icon-sprites.positioning')
1134
32
css_template_file = os.path.join(icing, 'style-3-0.css.in')
1135
33
css_file = os.path.join(icing, 'build/style-3-0.css')
1136
34
1137
35
sprite_util = SpriteUtil(
1138
36
    css_template_file, 'icon-sprites',
1139
37
    url_prefix_substitutions={'/@@/': '../images/'})
1140
38
1141
39
if command == 'create-image':
1142
40
    sprite_util.combineImages(icing)
1143
41
    sprite_util.savePNG(combined_image_file)
1144
42
    sprite_util.savePositioning(positioning_file)
1145
43
elif command == 'create-css':
1146
44
    sprite_util.loadPositioning(positioning_file)
1147
45
    # The icing/icon-sprites file is relative to the css file
1148
46
    # in the icing/build/ directory.
1149
47
    sprite_util.saveConvertedCSS(css_file, '../icon-sprites')
1150
0
48
1151
=== modified file 'buildout-templates/bin/test.in'
1152
--- buildout-templates/bin/test.in	2009-09-17 17:42:25 +0000
1153
+++ buildout-templates/bin/test.in	2010-05-05 01:58:48 +0000
1154
@@ -1,4 +1,4 @@
1156
1
#!${buildout:executable}
1
#!${buildout:executable} -S
1157
2
##############################################################################
2
##############################################################################
1158
3
#
3
#
1159
4
# Copyright (c) 2004 Zope Corporation and Contributors.
4
# Copyright (c) 2004 Zope Corporation and Contributors.
1160
@@ -14,9 +14,20 @@
1161
14
##############################################################################
14
##############################################################################
1162
15
"""Test script
15
"""Test script
1163
16
"""
16
"""
1167
17
import sys, os, time, logging, warnings, re
17
1168
18
18
# NOTE: This is a generated file.  The original is in
1169
19
BUILD_DIR = '${buildout:directory}'
19
# buildout-templates/bin/test.in
1170
20
1171
21
import logging, os, re, sys, time, warnings
1172
22
1173
23
# Initialize our paths.
1174
24
${python-relative-path-setup}
1175
25
import sys
1176
26
sys.path.insert(0, ${scripts:parts-directory|path-repr})
1177
27
import site
1178
28
1179
29
BUILD_DIR = ${buildout:directory|path-repr}
1180
30
CUSTOM_SITE_DIR = ${scripts:parts-directory|path-repr}
1181
20
31
1182
21
if os.getsid(0) == os.getsid(os.getppid()):
32
if os.getsid(0) == os.getsid(os.getppid()):
1183
22
    # We need to become the process group leader so test_on_merge.py
33
    # We need to become the process group leader so test_on_merge.py
1184
@@ -30,17 +41,16 @@
1185
30
41
1186
31
# Make tests run in a timezone no launchpad developers live in.
42
# Make tests run in a timezone no launchpad developers live in.
1187
32
# Our tests need to run in any timezone.
43
# Our tests need to run in any timezone.
1189
33
# (No longer actually required, as PQM does this)
44
# (This is no longer actually required, as PQM does this.)
1190
34
os.environ['TZ'] = 'Asia/Calcutta'
45
os.environ['TZ'] = 'Asia/Calcutta'
1191
35
time.tzset()
46
time.tzset()
1192
36
47
1200
37
# Enable Storm's C extensions
48
# Storm's C extensions should already be enabled from lp_sitecustomize.py,
1201
38
os.environ['STORM_CEXTENSIONS'] = '1'
49
# which our custom sitecustomize.py ran.
1202
39
50
assert os.environ['STORM_CEXTENSIONS'] == '1'
1203
40
sys.path[0:0] = [${string-paths}]
51
1204
41
52
# Make sure our site.py is the one that subprocesses use.
1205
42
# Set PYTHONPATH environment variable for spawned processes
53
os.environ['PYTHONPATH'] = CUSTOM_SITE_DIR
1199
43
os.environ['PYTHONPATH'] = ':'.join(sys.path)
1206
44
54
1207
45
# Set a flag if this is the main testrunner process
55
# Set a flag if this is the main testrunner process
1208
46
if len(sys.argv) > 1 and sys.argv[1] == '--resume-layer':
56
if len(sys.argv) > 1 and sys.argv[1] == '--resume-layer':
1209
@@ -88,14 +98,28 @@
1210
88
# need to be silenced should have an accompanied Bug reference.
98
# need to be silenced should have an accompanied Bug reference.
1211
89
#
99
#
1212
90
warnings.filterwarnings(
100
warnings.filterwarnings(
1214
91
    'ignore', 'PyCrypto', RuntimeWarning, 'twisted[.]conch[.]ssh'
101
    'ignore', 'PyCrypto', RuntimeWarning, 'twisted[.]conch[.]ssh',
1215
92
    )
102
    )
1216
93
warnings.filterwarnings(
103
warnings.filterwarnings(
1217
94
    'ignore', 'twisted.python.plugin', DeprecationWarning,
104
    'ignore', 'twisted.python.plugin', DeprecationWarning,
1218
95
    )
105
    )
1219
96
warnings.filterwarnings(
106
warnings.filterwarnings(
1222
97
    'ignore', 'bzrlib.*was deprecated', DeprecationWarning
107
    'ignore', 'zope.testing.doctest', DeprecationWarning,
1223
98
)
108
    )
1224
109
warnings.filterwarnings(
1225
110
    'ignore', 'bzrlib.*was deprecated', DeprecationWarning,
1226
111
    )
1227
112
# XXX: 2010-04-26, Salgado, bug=570246: Silence python2.6 deprecation
1228
113
# warnings.
1229
114
# We cannot narrow this warnings filter to just twisted because
1230
115
# warnings.warn_explicit() sees this import as coming from importfascist, not
1231
116
# from twisted.  It makes no sense to put module='importfascist' here though
1232
117
# because /everything/ gets imported through it.  So, sad as it is, until
1233
118
# twisted doesn't produce warnings under Python 2.6, just ignore all these
1234
119
# deprecations.
1235
120
warnings.filterwarnings(
1236
121
    'ignore', '.*(md5|sha|sets)', DeprecationWarning,
1237
122
    )
1238
99
# The next one is caused by a lamosity in python-openid.  The following change
123
# The next one is caused by a lamosity in python-openid.  The following change
1239
100
# to openid/server/server.py would make the warning filter unnecessary:
124
# to openid/server/server.py would make the warning filter unnecessary:
1240
101
# 978c974,974
125
# 978c974,974
1241
@@ -137,9 +161,9 @@
1242
137
defaults = {
161
defaults = {
1243
138
    # Find tests in the tests and ftests directories
162
    # Find tests in the tests and ftests directories
1244
139
    'tests_pattern': '^f?tests$',
163
    'tests_pattern': '^f?tests$',
1246
140
    'test_path': ['${buildout:directory}/lib'],
164
    'test_path': [${buildout:directory/lib|path-repr}],
1247
141
    'package': ['canonical', 'lp', 'devscripts'],
165
    'package': ['canonical', 'lp', 'devscripts'],
1249
142
    'layer': ['!(MailmanLayer|WindmillLayer)'],
166
    'layer': ['!(MailmanLayer)'],
1250
143
    }
167
    }
1251
144
168
1252
145
# Monkey-patch os.listdir to randomise the results
169
# Monkey-patch os.listdir to randomise the results
1253
@@ -161,13 +185,7 @@
1254
161
os.listdir = listdir
185
os.listdir = listdir
1255
162
186
1256
163
187
1264
164
from canonical.testing.customresult import (
188
from canonical.testing.customresult import filter_tests, patch_find_tests
1258
165
    filter_tests,
1259
166
    list_tests,
1260
167
    patch_find_tests,
1261
168
    patch_zope_testresult,
1262
169
    )
1263
170
from subunit import TestProtocolClient
1265
171
189
1266
172
190
1267
173
if __name__ == '__main__':
191
if __name__ == '__main__':
1268
@@ -191,16 +209,6 @@
1269
191
    options.parser.add_option(
209
    options.parser.add_option(
1270
192
        '--load-list', type=str, action='callback', callback=load_list)
210
        '--load-list', type=str, action='callback', callback=load_list)
1271
193
211
1272
194
    def list_test_option(option, opt, value, parser):
1273
195
        patch_find_tests(list_tests)
1274
196
    options.parser.add_option(
1275
197
        '--list', action='callback', callback=list_test_option)
1276
198
1277
199
    def use_subunit(option, opt, value, parser):
1278
200
        patch_zope_testresult(TestProtocolClient(sys.stdout))
1279
201
    options.parser.add_option(
1280
202
        '--subunit', action='callback', callback=use_subunit)
1281
203
1282
204
    # tests_pattern is a regexp, so the parsed value is hard to compare
212
    # tests_pattern is a regexp, so the parsed value is hard to compare
1283
205
    # with the default value in the loop below.
213
    # with the default value in the loop below.
1284
206
    options.parser.defaults['tests_pattern'] = defaults['tests_pattern']
214
    options.parser.defaults['tests_pattern'] = defaults['tests_pattern']
1285
@@ -229,7 +237,7 @@
1286
229
    # e.g. run the test that you are currently editing.
237
    # e.g. run the test that you are currently editing.
1287
230
    try:
238
    try:
1288
231
        there = os.getcwd()
239
        there = os.getcwd()
1290
232
        os.chdir('${buildout:directory}')
240
        os.chdir(BUILD_DIR)
1291
233
        result = testrunner.run([])
241
        result = testrunner.run([])
1292
234
    finally:
242
    finally:
1293
235
        os.chdir(there)
243
        os.chdir(there)
1294
236
244
1295
=== modified file 'buildout-templates/bin/update-download-cache.in'
1296
--- buildout-templates/bin/update-download-cache.in	2009-05-13 00:29:48 +0000
1297
+++ buildout-templates/bin/update-download-cache.in	2010-05-05 01:58:48 +0000
1298
@@ -1,2 +1,4 @@
1300
1
bzr up ${buildout:directory}/buildout/download-cache
1
${shell-relative-path-setup}
1301
2
1302
3
bzr up ${buildout:directory/buildout/download-cache|shell-path}
1303
2
4
1304
3
5
1305
=== modified file 'buildout.cfg'
1306
--- buildout.cfg	2009-09-21 10:45:15 +0000
1307
+++ buildout.cfg	2010-05-05 01:58:48 +0000
1308
@@ -11,6 +11,7 @@
1309
11
unzip = true
11
unzip = true
1310
12
eggs-directory = eggs
12
eggs-directory = eggs
1311
13
download-cache = download-cache
13
download-cache = download-cache
1312
14
relative-paths = true
1313
14
15
1314
15
# Disable this option temporarily if you want buildout to find software
16
# Disable this option temporarily if you want buildout to find software
1315
16
# dependencies *other* than those in our download-cache.  Once you have the
17
# dependencies *other* than those in our download-cache.  Once you have the
1316
@@ -25,9 +26,6 @@
1317
25
26
1318
26
allow-picked-versions = false
27
allow-picked-versions = false
1319
27
28
1320
28
allowed-eggs-from-site-packages =
1321
29
include-site-packages-for-buildout = false
1322
30
1323
31
prefer-final = true
29
prefer-final = true
1324
32
30
1325
33
develop = .
31
develop = .
1326
@@ -37,42 +35,41 @@
1327
37
35
1328
38
[filetemplates]
36
[filetemplates]
1329
39
recipe = z3c.recipe.filetemplate
37
recipe = z3c.recipe.filetemplate
1330
40
eggs = lp
1331
41
       windmill
1332
42
# XXX gary 2009-5-12 bug 375751:
1333
43
# Make mailman built and installed in a more normal way.
1334
44
extra-paths = ${buildout:directory}/lib/mailman
1335
45
source-directory = buildout-templates
38
source-directory = buildout-templates
1336
46
39
1337
47
[scripts]
40
[scripts]
1339
48
recipe = zc.recipe.egg
41
recipe = z3c.recipe.scripts
1340
49
eggs = lp
42
eggs = lp
1341
50
    windmill
43
    windmill
1342
51
    funkload
44
    funkload
1343
45
    zc.zservertracelog
1344
52
# XXX gary 2009-5-12 bug 375751:
46
# XXX gary 2009-5-12 bug 375751:
1345
53
# Make mailman built and installed in a more normal way.
47
# Make mailman built and installed in a more normal way.
1346
54
extra-paths = ${buildout:directory}/lib/mailman
48
extra-paths = ${buildout:directory}/lib/mailman
1358
55
# note that any indentation is lost in initialization blocks
49
include-site-packages = true
1359
56
initialization = import os
50
allowed-eggs-from-site-packages =
1360
57
                 os.environ['STORM_CEXTENSIONS'] = '1'
51
interpreter = py
1361
58
                 os.environ.setdefault('LPCONFIG', '${configuration:instance_name}')
52
# Note that any indentation is lost in initialization blocks.
1362
59
                 # XXX 2009-08-21 gary bug 417077
53
initialization =
1363
60
                 # This can hopefully be removed when Twisted is used as an egg.
54
    import os
1364
61
                 import warnings
55
    os.environ.setdefault('LPCONFIG', '${configuration:instance_name}')
1365
62
                 warnings.filterwarnings(
56
    import lp_sitecustomize # Initializes LP environment.
1355
63
                 'ignore',
1356
64
                 'Module .+ was already imported from .+, but .+ is being added.*',
1357
65
                 UserWarning)
1366
66
entry-points = stxdocs=zope.configuration.stxdocs:main
57
entry-points = stxdocs=zope.configuration.stxdocs:main
1367
67
    googletestservice=canonical.launchpad.testing.googletestservice:main
58
    googletestservice=canonical.launchpad.testing.googletestservice:main
1368
68
    windmill=windmill.bin.windmill_bin:main
59
    windmill=windmill.bin.windmill_bin:main
1369
69
    lp-windmill=lp.scripts.utilities.lpwindmill:main
60
    lp-windmill=lp.scripts.utilities.lpwindmill:main
1370
61
    jsbuild=lazr.js.build:main
1371
62
    jslint=lazr.js.jslint:main
1372
63
    tracereport=zc.zservertracelog.tracereport:main
1373
64
    jssize=lp.scripts.utilities.jssize:main
1374
70
65
1375
71
[iharness]
66
[iharness]
1377
72
recipe = zc.recipe.egg
67
recipe = z3c.recipe.scripts
1378
73
eggs = ${scripts:eggs}
68
eggs = ${scripts:eggs}
1379
74
     ipython
69
     ipython
1380
75
extra-paths = ${scripts:extra-paths}
70
extra-paths = ${scripts:extra-paths}
1381
71
include-site-packages = true
1382
72
allowed-eggs-from-site-packages =
1383
76
initialization = ${scripts:initialization}
73
initialization = ${scripts:initialization}
1384
77
entry-points = iharness=canonical.database.harness:ipython
74
entry-points = iharness=canonical.database.harness:ipython
1385
78
scripts = iharness ipython=ipy
75
scripts = iharness ipython=ipy
1386
79
76
1387
=== added symlink 'bzrplugins/builder'
1388
=== target is u'../sourcecode/bzr-builder/'
1389
=== modified file 'bzrplugins/lpserve.py'
1390
--- bzrplugins/lpserve.py	2009-07-17 00:26:05 +0000
1391
+++ bzrplugins/lpserve.py	2010-05-05 01:58:48 +0000
1392
@@ -11,6 +11,7 @@
1393
11
__all__ = ['cmd_launchpad_server']
11
__all__ = ['cmd_launchpad_server']
1394
12
12
1395
13
13
1396
14
import resource
1397
14
import sys
15
import sys
1398
15
16
1399
16
from bzrlib.commands import Command, register_command
17
from bzrlib.commands import Command, register_command
1400
@@ -44,9 +45,9 @@
1401
44
        Option('mirror-directory',
45
        Option('mirror-directory',
1402
45
               help='serve branches from this directory. Defaults to '
46
               help='serve branches from this directory. Defaults to '
1403
46
                    'config.codehosting.mirrored_branches_root.'),
47
                    'config.codehosting.mirrored_branches_root.'),
1405
47
        Option('branchfs-endpoint',
48
        Option('codehosting-endpoint',
1406
48
               help='the url of the internal XML-RPC server. Defaults to '
49
               help='the url of the internal XML-RPC server. Defaults to '
1408
49
                    'config.codehosting.branchfs_endpoint.',
50
                    'config.codehosting.codehosting_endpoint.',
1409
50
               type=unicode),
51
               type=unicode),
1410
51
        ]
52
        ]
1411
52
53
1412
@@ -84,15 +85,18 @@
1413
84
        finally:
85
        finally:
1414
85
            ui.ui_factory = old_factory
86
            ui.ui_factory = old_factory
1415
86
87
1418
87
    def run(self, user_id, port=None, upload_directory=None,
88
    def run(self, user_id, port=None, branch_directory=None,
1419
88
            mirror_directory=None, branchfs_endpoint_url=None, inet=False):
89
            codehosting_endpoint_url=None, inet=False):
1420
89
        from lp.codehosting.bzrutils import install_oops_handler
90
        from lp.codehosting.bzrutils import install_oops_handler
1422
90
        from lp.codehosting.vfs import get_lp_server
91
        from lp.codehosting.vfs import get_lp_server, hooks
1423
91
        install_oops_handler(user_id)
92
        install_oops_handler(user_id)
1424
93
        four_gig = int(4e9)
1425
94
        resource.setrlimit(resource.RLIMIT_AS, (four_gig, four_gig))
1426
95
        seen_new_branch = hooks.SetProcTitleHook()
1427
92
        lp_server = get_lp_server(
96
        lp_server = get_lp_server(
1431
93
            int(user_id), branchfs_endpoint_url,
97
            int(user_id), codehosting_endpoint_url, branch_directory,
1432
94
            upload_directory, mirror_directory)
98
            seen_new_branch.seen)
1433
95
        lp_server.setUp()
99
        lp_server.start_server()
1434
96
100
1435
97
        old_lockdir_timeout = lockdir._DEFAULT_TIMEOUT_SECONDS
101
        old_lockdir_timeout = lockdir._DEFAULT_TIMEOUT_SECONDS
1436
98
        try:
102
        try:
1437
@@ -102,7 +106,7 @@
1438
102
            self.run_server(smart_server)
106
            self.run_server(smart_server)
1439
103
        finally:
107
        finally:
1440
104
            lockdir._DEFAULT_TIMEOUT_SECONDS = old_lockdir_timeout
108
            lockdir._DEFAULT_TIMEOUT_SECONDS = old_lockdir_timeout
1442
105
            lp_server.tearDown()
109
            lp_server.stop_server()
1443
106
110
1444
107
111
1445
108
register_command(cmd_launchpad_server)
112
register_command(cmd_launchpad_server)
1446
109
113
1447
=== added file 'configs/development/build-from-branch.zcml'
1448
--- configs/development/build-from-branch.zcml	1970-01-01 00:00:00 +0000
1449
+++ configs/development/build-from-branch.zcml	2010-05-05 01:58:48 +0000
1450
@@ -0,0 +1,119 @@
1451
1
<!-- Copyright 2010 Canonical Ltd.  This software is licensed under the
1452
2
     GNU Affero General Public License version 3 (see the file LICENSE).
1453
3
-->
1454
4
1455
5
<configure
1456
6
    xmlns="http://namespaces.zope.org/zope"
1457
7
    xmlns:browser="http://namespaces.zope.org/browser"
1458
8
    xmlns:i18n="http://namespaces.zope.org/i18n"
1459
9
    xmlns:xmlrpc="http://namespaces.zope.org/xmlrpc"
1460
10
    i18n_domain="launchpad">
1461
11
    <facet facet="branches">
1462
12
1463
13
        <browser:defaultView
1464
14
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1465
15
            name="+index"
1466
16
            layer="canonical.launchpad.layers.CodeLayer"/>
1467
17
        <browser:page
1468
18
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1469
19
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeView"
1470
20
            name="+index"
1471
21
            template="../../lib/lp/code/templates/sourcepackagerecipe-index.pt"
1472
22
            permission="launchpad.View"/>
1473
23
        <browser:page
1474
24
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1475
25
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeRequestBuildsView"
1476
26
            name="+request-builds"
1477
27
            template="../../lib/lp/code/templates/sourcepackagerecipe-request-builds.pt"
1478
28
            permission="launchpad.View"/>
1479
29
    </facet>
1480
30
    <facet facet="branches">
1481
31
        <browser:defaultView
1482
32
            for="lp.code.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuild"
1483
33
            name="+index"
1484
34
            layer="canonical.launchpad.layers.CodeLayer"/>
1485
35
        <browser:page
1486
36
            for="lp.code.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuild"
1487
37
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeBuildView"
1488
38
            name="+index"
1489
39
            template="../../lib/lp/code/templates/sourcepackagerecipe-index.pt"
1490
40
            permission="launchpad.View"/>
1491
41
        <browser:menus
1492
42
            classes="
1493
43
                SourcePackageRecipeNavigationMenu
1494
44
                SourcePackageRecipeContextMenu"
1495
45
            module="lp.code.browser.sourcepackagerecipe"/>
1496
46
1497
47
        <browser:page
1498
48
            for="lp.code.interfaces.branch.IBranch"
1499
49
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeAddView"
1500
50
            permission="zope.Public"
1501
51
            facet="branches"
1502
52
            name="+new-recipe"
1503
53
            template="../../lib/lp/app/templates/generic-edit.pt"/>
1504
54
        <browser:page
1505
55
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1506
56
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeEditView"
1507
57
            permission="zope.Public"
1508
58
            facet="branches"
1509
59
            name="+edit"
1510
60
            template="../../lib/lp/app/templates/generic-edit.pt"/>
1511
61
        <browser:page
1512
62
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1513
63
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeDeleteView"
1514
64
            permission="zope.Public"
1515
65
            facet="branches"
1516
66
            name="+delete"
1517
67
            template="../../lib/lp/app/templates/generic-edit.pt"/>
1518
68
1519
69
1520
70
        <browser:page
1521
71
            for="lp.code.interfaces.branch.IBranch"
1522
72
            class="lp.code.browser.sourcepackagerecipelisting.BranchRecipeListingView"
1523
73
            permission="zope.Public"
1524
74
            facet="branches"
1525
75
            name="+recipes"
1526
76
            template="../../lib/lp/code/templates/sourcepackagerecipe-listing.pt"/>
1527
77
        <browser:page
1528
78
            for="lp.registry.interfaces.product.IProduct"
1529
79
            class="lp.code.browser.sourcepackagerecipelisting.ProductRecipeListingView"
1530
80
            permission="zope.Public"
1531
81
            facet="branches"
1532
82
            name="+recipes"
1533
83
            template="../../lib/lp/code/templates/sourcepackagerecipe-listing.pt"/>
1534
84
        <browser:page
1535
85
            for="lp.registry.interfaces.person.IPerson"
1536
86
            class="lp.code.browser.sourcepackagerecipelisting.PersonRecipeListingView"
1537
87
            permission="zope.Public"
1538
88
            facet="branches"
1539
89
            name="+recipes"
1540
90
            template="../../lib/lp/code/templates/sourcepackagerecipe-listing.pt"/>
1541
91
1542
92
        <browser:page
1543
93
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1544
94
            name="+hierarchy"
1545
95
            class="lp.code.browser.sourcepackagerecipe.SourcePackageRecipeHierarchy"
1546
96
            template="../../lib/lp/app/templates/launchpad-hierarchy.pt"
1547
97
            permission="zope.Public"/>
1548
98
1549
99
        <adapter
1550
100
            provides="canonical.launchpad.webapp.interfaces.IBreadcrumb"
1551
101
            for="lp.code.interfaces.sourcepackagerecipe.ISourcePackageRecipe"
1552
102
            factory="canonical.launchpad.webapp.breadcrumb.NameBreadcrumb"
1553
103
            permission="zope.Public"/>
1554
104
    </facet>
1555
105
  <securedutility
1556
106
    name="BuildableDistroSeries"
1557
107
    component="lp.code.browser.sourcepackagerecipe.buildable_distroseries_vocabulary"
1558
108
    provides="zope.schema.interfaces.IVocabularyFactory"
1559
109
    >
1560
110
    <allow interface="zope.schema.interfaces.IVocabularyFactory"/>
1561
111
  </securedutility>
1562
112
  <securedutility
1563
113
    name="TargetPPAs"
1564
114
    component="lp.code.browser.sourcepackagerecipe.target_ppas_vocabulary"
1565
115
    provides="zope.schema.interfaces.IVocabularyFactory"
1566
116
    >
1567
117
    <allow interface="zope.schema.interfaces.IVocabularyFactory"/>
1568
118
  </securedutility>
1569
119
</configure>
1570
0
120
1571
=== modified file 'configs/development/launchpad-lazr.conf'
1572
--- configs/development/launchpad-lazr.conf	2009-09-29 15:15:21 +0000
1573
+++ configs/development/launchpad-lazr.conf	2010-05-05 01:58:48 +0000
1574
@@ -7,6 +7,7 @@
1575
7
7
1576
8
[archivepublisher]
8
[archivepublisher]
1577
9
root: /var/tmp/archive
9
root: /var/tmp/archive
1578
10
base_url: http://archive.launchpad.dev/
1579
10
11
1580
11
[branchscanner]
12
[branchscanner]
1581
12
oops_prefix: BS
13
oops_prefix: BS
1582
@@ -14,7 +15,8 @@
1583
14
15
1584
15
[builddmaster]
16
[builddmaster]
1585
16
root: /var/tmp/builddmaster/
17
root: /var/tmp/builddmaster/
1587
17
uploader: /bin/echo Uploader invocation of build BUILDID in:
18
uploader: scripts/process-upload.py -Mvv
1588
19
bzr_builder_sources_list: None
1589
18
20
1590
19
[buildsequencer]
21
[buildsequencer]
1591
20
mailproblemsto: root
22
mailproblemsto: root
1592
@@ -44,15 +46,14 @@
1593
44
46
1594
45
[codebrowse]
47
[codebrowse]
1595
46
cachepath: /var/tmp/codebrowse.launchpad.dev/cache
48
cachepath: /var/tmp/codebrowse.launchpad.dev/cache
1597
47
log_folder:
49
log_folder: /var/tmp/codebrowse.launchpad.dev/logs
1598
48
launchpad_root: https://code.launchpad.dev/
50
launchpad_root: https://code.launchpad.dev/
1599
49
secret_path: configs/development/codebrowse-secret
51
secret_path: configs/development/codebrowse-secret
1600
50
52
1601
51
[codehosting]
53
[codehosting]
1602
52
launch: True
54
launch: True
1603
53
authentication_endpoint: http://xmlrpc-private.launchpad.dev:8087/authserver
55
authentication_endpoint: http://xmlrpc-private.launchpad.dev:8087/authserver
1606
54
branchfs_endpoint: http://xmlrpc-private.launchpad.dev:8087/branchfilesystem
56
codehosting_endpoint: http://xmlrpc-private.launchpad.dev:8087/codehosting
1605
55
branch_puller_endpoint: http://xmlrpc-private.launchpad.dev:8087/branch_puller
1607
56
supermirror_root: http://bazaar.launchpad.dev/
57
supermirror_root: http://bazaar.launchpad.dev/
1608
57
hosted_branches_root: /var/tmp/bazaar.launchpad.dev/push-branches/
58
hosted_branches_root: /var/tmp/bazaar.launchpad.dev/push-branches/
1609
58
codebrowse_root: http://bazaar.launchpad.dev/
59
codebrowse_root: http://bazaar.launchpad.dev/
1610
@@ -90,10 +91,17 @@
1611
90
91
1612
91
92
1613
92
[database]
93
[database]
1618
93
main_master: dbname=launchpad_dev
94
rw_main_master: dbname=launchpad_dev
1619
94
main_slave:  dbname=launchpad_dev
95
rw_main_slave:  dbname=launchpad_dev
1620
95
auth_master: dbname=launchpad_dev
96
# Use our _template databases here just so that we have different values from
1621
96
auth_slave:  dbname=launchpad_dev
97
# the rw_* configs.
1622
98
ro_main_master: dbname=launchpad_dev_template
1623
99
ro_main_slave:  dbname=launchpad_dev_template
1624
100
1625
101
# XXX stub 20100407 bug=557271: These next two are ignored, and should
1626
102
# be removed after the May 2010 rollout.
1627
103
auth_master: bug 557271
1628
104
auth_slave:  bug 557271
1629
97
105
1630
98
[distributionmirrorprober]
106
[distributionmirrorprober]
1631
99
use_proxy: False
107
use_proxy: False
1632
@@ -119,18 +127,21 @@
1633
119
public_host: keyserver.launchpad.dev
127
public_host: keyserver.launchpad.dev
1634
120
128
1635
121
[launchpad]
129
[launchpad]
1636
130
enable_test_openid_provider: True
1637
131
openid_provider_vhost: testopenid
1638
122
code_domain: code.launchpad.dev
132
code_domain: code.launchpad.dev
1639
123
default_batch_size: 5
133
default_batch_size: 5
1640
124
max_attachment_size: 2097152
134
max_attachment_size: 2097152
1641
125
branchlisting_batch_size: 6
135
branchlisting_batch_size: 6
1642
136
mugshot_batch_size: 8
1643
137
announcement_batch_size: 4
1644
138
download_batch_size: 4
1645
126
openid_preauthorization_acl:
139
openid_preauthorization_acl:
1646
127
    localhost http://launchpad.dev/
140
    localhost http://launchpad.dev/
1647
128
max_bug_feed_cache_minutes: 30
141
max_bug_feed_cache_minutes: 30
1648
129
bzr_imports_root_url: file:///tmp/bazaar-branches
142
bzr_imports_root_url: file:///tmp/bazaar-branches
1649
130
geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
143
geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
1650
131
geonames_identity: lpdev
144
geonames_identity: lpdev
1651
132
# Set to True to test read-only mode.
1652
133
read_only: False
1653
134
storm_cache: generational
145
storm_cache: generational
1654
135
storm_cache_size: 100
146
storm_cache_size: 100
1655
136
147
1656
@@ -180,6 +191,20 @@
1657
180
soft_max_size: 40000
191
soft_max_size: 40000
1658
181
hard_max_size: 1000000
192
hard_max_size: 1000000
1659
182
193
1660
194
[memcache]
1661
195
servers: (127.0.0.1:11217,1)
1662
196
1663
197
[memcached]
1664
198
launch: True
1665
199
verbose: False
1666
200
address: 127.0.0.1
1667
201
port: 11217
1668
202
memory_size: 1
1669
203
1670
204
[merge_proposal_jobs]
1671
205
error_dir: /var/tmp/codehosting.test
1672
206
oops_prefix: DMPJ
1673
207
1674
183
[personalpackagearchive]
208
[personalpackagearchive]
1675
184
root: /var/tmp/ppa/
209
root: /var/tmp/ppa/
1676
185
private_root: /var/tmp/ppa
210
private_root: /var/tmp/ppa
1677
@@ -192,6 +217,7 @@
1678
192
217
1679
193
[rosetta]
218
[rosetta]
1680
194
global_suggestions_enabled: True
219
global_suggestions_enabled: True
1681
220
generate_templates: True
1682
195
221
1683
196
[rosettabranches]
222
[rosettabranches]
1684
197
error_dir: /var/tmp/rosettabranches.test
223
error_dir: /var/tmp/rosettabranches.test
1685
@@ -201,6 +227,9 @@
1686
201
error_dir: /var/tmp/poimport
227
error_dir: /var/tmp/poimport
1687
202
oops_prefix: POI
228
oops_prefix: POI
1688
203
229
1689
230
[process_apport_blobs]
1690
231
error_dir: /var/tmp/lperr
1691
232
1692
204
[supermirror_puller]
233
[supermirror_puller]
1693
205
error_dir: /var/tmp/codehosting.test
234
error_dir: /var/tmp/codehosting.test
1694
206
oops_prefix: SMP
235
oops_prefix: SMP
1695
@@ -217,6 +246,10 @@
1696
217
error_dir: /var/tmp/codehosting.test
246
error_dir: /var/tmp/codehosting.test
1697
218
oops_prefix: USMP
247
oops_prefix: USMP
1698
219
248
1699
249
[upgrade_branches]
1700
250
oops_prefix: UBJD
1701
251
error_dir: /var/tmp/codehosting.test
1702
252
1703
220
[uploader]
253
[uploader]
1704
221
default_recipient_name: Local Root
254
default_recipient_name: Local Root
1705
222
default_sender_address: root@localhost
255
default_sender_address: root@localhost
1706
@@ -233,7 +266,7 @@
1707
233
266
1708
234
[vhost.api]
267
[vhost.api]
1709
235
hostname: api.launchpad.dev
268
hostname: api.launchpad.dev
1711
236
rooturl: https://api.launchpad.dev/beta/
269
rooturl: https://api.launchpad.dev/
1712
237
270
1713
238
[vhost.blueprints]
271
[vhost.blueprints]
1714
239
hostname: blueprints.launchpad.dev
272
hostname: blueprints.launchpad.dev
1715
@@ -253,6 +286,12 @@
1716
253
[vhost.openid]
286
[vhost.openid]
1717
254
hostname: openid.launchpad.dev
287
hostname: openid.launchpad.dev
1718
255
288
1719
289
[vhost.testopenid]
1720
290
hostname: testopenid.dev
1721
291
1722
292
[vhost.ubuntu_openid]
1723
293
hostname: ubuntu-openid.launchpad.dev
1724
294
1725
256
[vhost.shipitubuntu]
295
[vhost.shipitubuntu]
1726
257
hostname: shipit.ubuntu.dev
296
hostname: shipit.ubuntu.dev
1727
258
297
1728
@@ -278,3 +317,6 @@
1729
278
# so disable that here. note that the testrunner config inherits
317
# so disable that here. note that the testrunner config inherits
1730
279
# this setting from us.
318
# this setting from us.
1731
280
send_email: false
319
send_email: false
1732
320
1733
321
[build_from_branch]
1734
322
enabled = True
1735
281
323
1736
=== modified file 'configs/development/local-launchpad-apache'
1737
--- configs/development/local-launchpad-apache	2009-07-24 01:57:06 +0000
1738
+++ configs/development/local-launchpad-apache	2010-05-05 01:58:48 +0000
1739
@@ -112,6 +112,19 @@
1740
112
  </Directory>
112
  </Directory>
1741
113
</VirtualHost>
113
</VirtualHost>
1742
114
114
1743
115
<VirtualHost 127.0.0.88:80>
1744
116
  ServerName archive.launchpad.dev
1745
117
  LogLevel debug
1746
118
1747
119
  DocumentRoot /var/tmp/archive
1748
120
  <Directory /var/tmp/archive/>
1749
121
    Order Deny,Allow
1750
122
    Deny from all
1751
123
    Allow from 127.0.0.0/255.0.0.0
1752
124
    Options Indexes
1753
125
  </Directory>
1754
126
</VirtualHost>
1755
127
1756
115
<VirtualHost 127.0.0.88:443>
128
<VirtualHost 127.0.0.88:443>
1757
116
  ServerName launchpad.dev
129
  ServerName launchpad.dev
1758
117
  ServerAlias *.launchpad.dev
130
  ServerAlias *.launchpad.dev
1759
118
131
1760
=== modified file 'configs/replicated-development/launchpad-lazr.conf'
1761
--- configs/replicated-development/launchpad-lazr.conf	2008-10-14 11:10:35 +0000
1762
+++ configs/replicated-development/launchpad-lazr.conf	2010-05-05 01:58:48 +0000
1763
@@ -6,8 +6,7 @@
1764
6
extends: ../development/launchpad-lazr.conf
6
extends: ../development/launchpad-lazr.conf
1765
7
7
1766
8
[database]
8
[database]
1772
9
main_master: dbname=launchpad_dev
9
rw_main_master: dbname=launchpad_dev
1773
10
main_slave: dbname=launchpad_dev_slave
10
rw_main_slave: dbname=launchpad_dev_slave
1774
11
auth_master: dbname=launchpad_dev
11
ro_main_master: dbname=launchpad_dev
1775
12
auth_slave: dbname=launchpad_dev_slave
12
ro_main_slave: dbname=launchpad_dev_slave
1771
13
1776
14
13
1777
=== modified file 'configs/test-playground/launchpad-lazr.conf'
1778
--- configs/test-playground/launchpad-lazr.conf	2008-11-10 16:12:10 +0000
1779
+++ configs/test-playground/launchpad-lazr.conf	2010-05-05 01:58:48 +0000
1780
@@ -6,7 +6,7 @@
1781
6
extends: ../development/launchpad-lazr.conf
6
extends: ../development/launchpad-lazr.conf
1782
7
7
1783
8
[database]
8
[database]
1788
9
main_master: dbname=launchpad_ftest_playground
9
rw_main_master: dbname=launchpad_ftest_playground
1789
10
main_slave:  dbname=launchpad_ftest_playground
10
rw_main_slave:  dbname=launchpad_ftest_playground
1790
11
auth_master: dbname=launchpad_ftest_playground
11
ro_main_master: dbname=launchpad_ftest_playground
1791
12
auth_slave:  dbname=launchpad_ftest_playground
12
ro_main_slave:  dbname=launchpad_ftest_playground
1792
13
13
1793
=== modified file 'configs/testrunner-appserver/launchpad-lazr.conf'
1794
--- configs/testrunner-appserver/launchpad-lazr.conf	2009-04-29 19:10:17 +0000
1795
+++ configs/testrunner-appserver/launchpad-lazr.conf	2010-05-05 01:58:48 +0000
1796
@@ -23,7 +23,7 @@
1797
23
rooturl: http://launchpad.dev:8085/
23
rooturl: http://launchpad.dev:8085/
1798
24
24
1799
25
[vhost.api]
25
[vhost.api]
1801
26
rooturl: http://api.launchpad.dev:8085/beta/
26
rooturl: http://api.launchpad.dev:8085/
1802
27
27
1803
28
[vhost.blueprints]
28
[vhost.blueprints]
1804
29
rooturl: http://blueprints.launchpad.dev:8085/
29
rooturl: http://blueprints.launchpad.dev:8085/
1805
@@ -43,6 +43,9 @@
1806
43
[vhost.openid]
43
[vhost.openid]
1807
44
rooturl: http://openid.launchpad.dev:8085/
44
rooturl: http://openid.launchpad.dev:8085/
1808
45
45
1809
46
[vhost.testopenid]
1810
47
rooturl: http://testopenid.dev:8085/
1811
48
1812
46
[vhost.shipitubuntu]
49
[vhost.shipitubuntu]
1813
47
rooturl: http://shipit.ubuntu.dev:8085/
50
rooturl: http://shipit.ubuntu.dev:8085/
1814
48
51
1815
49
52
1816
=== added file 'configs/testrunner-appserver/yui-unittest.zcml'
1817
--- configs/testrunner-appserver/yui-unittest.zcml	1970-01-01 00:00:00 +0000
1818
+++ configs/testrunner-appserver/yui-unittest.zcml	2010-05-05 01:58:48 +0000
1819
@@ -0,0 +1,16 @@
1820
1
<!-- Copyright 2010 Canonical Ltd.  This software is licensed under the
1821
2
     GNU Affero General Public License version 3 (see the file LICENSE).
1822
3
-->
1823
4
1824
5
<configure
1825
6
    xmlns="http://namespaces.zope.org/zope"
1826
7
    xmlns:browser="http://namespaces.zope.org/browser">
1827
8
1828
9
    <browser:page
1829
10
        name="+yui-unittest"
1830
11
        for="canonical.launchpad.webapp.interfaces.ILaunchpadRoot"
1831
12
        class="lp.testing.views.YUITestFileView"
1832
13
        attribute="__call__"
1833
14
        permission="zope.Public"/>
1834
15
1835
16
</configure>
1836
0
17
1837
=== added symlink 'configs/testrunner/build-from-branch.zcml'
1838
=== target is u'../development/build-from-branch.zcml'
1839
=== modified file 'configs/testrunner/launchpad-lazr.conf'
1840
--- configs/testrunner/launchpad-lazr.conf	2009-09-29 15:15:21 +0000
1841
+++ configs/testrunner/launchpad-lazr.conf	2010-05-05 01:58:48 +0000
1842
@@ -8,13 +8,16 @@
1843
8
[canonical]
8
[canonical]
1844
9
chunkydiff: False
9
chunkydiff: False
1845
10
10
1846
11
[archivepublisher]
1847
12
base_url: http://ftpmaster.internal/
1848
13
1849
11
[branchscanner]
14
[branchscanner]
1850
12
oops_prefix: TSMS
15
oops_prefix: TSMS
1851
13
error_dir: /var/tmp/lperr.test
16
error_dir: /var/tmp/lperr.test
1852
14
17
1853
15
[builddmaster]
18
[builddmaster]
1854
16
socket_timeout: 10
19
socket_timeout: 10
1856
17
uploader: scripts/process-upload.py -Mvv --context buildd
20
uploader: scripts/process-upload.py -Mvv
1857
18
21
1858
19
[buildsequencer]
22
[buildsequencer]
1859
20
mailproblemsto: -
23
mailproblemsto: -
1860
@@ -31,21 +34,23 @@
1861
31
bzr_lp_prefix: lp://dev/
34
bzr_lp_prefix: lp://dev/
1862
32
hosted_branches_root: /tmp/sftp-test/branches
35
hosted_branches_root: /tmp/sftp-test/branches
1863
33
host_key_pair_path: lib/lp/codehosting/sshserver/tests/keys
36
host_key_pair_path: lib/lp/codehosting/sshserver/tests/keys
1865
34
port: tcp:22222:interface=127.0.0.1
37
port: tcp:22222:interface=bazaar.launchpad.dev
1866
35
error_dir: /var/tmp/codehosting.test
38
error_dir: /var/tmp/codehosting.test
1867
36
oops_prefix: SMPSSH
39
oops_prefix: SMPSSH
1868
37
access_log: /tmp/test-codehosting-access.log
40
access_log: /tmp/test-codehosting-access.log
1870
38
internal_branch_by_id_root: file:///var/tmp/bzrsync/
41
internal_branch_by_id_root: file:///var/tmp/bazaar.launchpad.dev/mirrors
1871
39
42
1872
40
[create_merge_proposals]
43
[create_merge_proposals]
1873
41
oops_prefix: TMPCJ
44
oops_prefix: TMPCJ
1874
42
error_dir: /var/tmp/codehosting.test
45
error_dir: /var/tmp/codehosting.test
1875
43
46
1876
44
[database]
47
[database]
1881
45
main_master: dbname=launchpad_ftest
48
rw_main_master: dbname=launchpad_ftest
1882
46
main_slave:  dbname=launchpad_ftest
49
rw_main_slave:  dbname=launchpad_ftest
1883
47
auth_master: dbname=launchpad_ftest
50
# Use our _template databases here just so that we have different values from
1884
48
auth_slave:  dbname=launchpad_ftest
51
# the rw_* configs.
1885
52
ro_main_master: dbname=launchpad_ftest_template
1886
53
ro_main_slave:  dbname=launchpad_ftest_template
1887
49
randomise_select_results: true
54
randomise_select_results: true
1888
50
55
1889
51
[error_reports]
56
[error_reports]
1890
@@ -122,7 +127,6 @@
1891
122
127
1892
123
[launchpad]
128
[launchpad]
1893
124
max_attachment_size: 1024
129
max_attachment_size: 1024
1894
125
bzr_imports_root_url: http://localhost:10899
1895
126
geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
130
geoip_database: /usr/share/GeoIP/GeoLiteCity.dat
1896
127
131
1897
128
[launchpad_session]
132
[launchpad_session]
1898
@@ -154,22 +158,45 @@
1899
154
bugnotification_interval: 5
158
bugnotification_interval: 5
1900
155
debbugs_db_location: lib/canonical/launchpad/components/ftests/debbugs_db
159
debbugs_db_location: lib/canonical/launchpad/components/ftests/debbugs_db
1901
156
160
1905
157
161
[memcache]
1906
158
[mpcreationjobs]
162
servers: (127.0.0.1:11242,1)
1907
159
oops_prefix: TMPCJ
163
1908
164
[memcached]
1909
165
# The test suite takes care of launching this as necessary.
1910
166
launch: false
1911
167
verbose: false
1912
168
memory_size: 1
1913
169
address: 127.0.0.1
1914
170
# We want a different port to ensure we don't pick up stray memcached
1915
171
# processes spawned through some other mechanism.
1916
172
port: 11242
1917
173
1918
174
[merge_proposal_jobs]
1919
175
oops_prefix: TMPJ
1920
160
error_dir: /var/tmp/codehosting.test
176
error_dir: /var/tmp/codehosting.test
1921
161
177
1924
162
[update_preview_diffs]
178
[upgrade_branches]
1925
163
oops_prefix: TUPD
179
oops_prefix: TUB
1926
164
error_dir: /var/tmp/codehosting.test
180
error_dir: /var/tmp/codehosting.test
1927
165
181
1928
166
[personalpackagearchive]
182
[personalpackagearchive]
1929
167
root: /var/tmp/ppa.test/
183
root: /var/tmp/ppa.test/
1930
168
184
1931
185
[ppa_apache_log_parser]
1932
186
logs_root: lib/lp/soyuz/scripts/tests/ppa-apache-log-files
1933
187
1934
169
[poimport]
188
[poimport]
1935
170
error_dir: /var/tmp/poimport.test
189
error_dir: /var/tmp/poimport.test
1936
171
oops_prefix: TPOI
190
oops_prefix: TPOI
1937
172
191
1938
192
[process_apport_blobs]
1939
193
dbuser: process-apport-blobs
1940
194
oops_prefix: TAPPORTBLOB
1941
195
error_dir: /var/tmp/lperr.test
1942
196
1943
197
[rosetta]
1944
198
generate_templates: True
1945
199
1946
173
[rosettabranches]
200
[rosettabranches]
1947
174
oops_prefix: TRSBR
201
oops_prefix: TRSBR
1948
175
error_dir: /var/tmp/rosettabranches.test
202
error_dir: /var/tmp/rosettabranches.test
1949
@@ -198,10 +225,13 @@
1950
198
rooturl: http://launchpad.dev/
225
rooturl: http://launchpad.dev/
1951
199
226
1952
200
[vhost.api]
227
[vhost.api]
1954
201
rooturl: http://api.launchpad.dev/beta/
228
rooturl: http://api.launchpad.dev/
1955
202
229
1956
203
[vhosts]
230
[vhosts]
1957
204
use_https: False
231
use_https: False
1958
205
232
1959
206
[zeca]
233
[zeca]
1960
207
root: /var/tmp/zeca.test
234
root: /var/tmp/zeca.test
1961
235
1962
236
[build_from_branch]
1963
237
enabled = True
1964
208
238
1965
=== modified file 'cronscripts/allocate-revision-karma.py'
1966
--- cronscripts/allocate-revision-karma.py	2009-06-24 20:52:01 +0000
1967
+++ cronscripts/allocate-revision-karma.py	2010-05-05 01:58:48 +0000
1968
@@ -1,4 +1,4 @@
1970
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
1971
2
#
2
#
1972
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
1973
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
1974
5
5
1975
=== removed file 'cronscripts/branch-scanner.py'
1976
--- cronscripts/branch-scanner.py	2009-06-24 20:52:01 +0000
1977
+++ cronscripts/branch-scanner.py	1970-01-01 00:00:00 +0000
1978
@@ -1,36 +0,0 @@
1979
1
#!/usr/bin/python2.4
1980
2
#
1981
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
1982
4
# GNU Affero General Public License version 3 (see the file LICENSE).
1983
5
1984
6
# pylint: disable-msg=C0103,W0403
1985
7
# Author: Gustavo Niemeyer <gustavo@niemeyer.net>
1986
8
#         David Allouche <david@allouche.net>
1987
9
1988
10
"""Update bzr branches information in the database"""
1989
11
1990
12
1991
13
import _pythonpath
1992
14
import logging
1993
15
1994
16
from lp.codehosting.scanner.branch_scanner import BranchScanner
1995
17
from canonical.config import config
1996
18
from lp.services.scripts.base import LaunchpadCronScript
1997
19
from canonical.launchpad.webapp.errorlog import globalErrorUtility
1998
20
1999
21
2000
22
class UpdateBranches(LaunchpadCronScript):
2001
23
    def main(self):
2002
24
        # We don't want debug messages from bzr at that point.
2003
25
        bzr_logger = logging.getLogger("bzr")
2004
26
        bzr_logger.setLevel(logging.INFO)
2005
27
        globalErrorUtility.configure('branchscanner')
2006
28
2007
29
        BranchScanner(self.txn, self.logger).scanAllBranches()
2008
30
2009
31
2010
32
if __name__ == '__main__':
2011
33
    script = UpdateBranches(
2012
34
        "updatebranches", dbuser=config.branchscanner.dbuser)
2013
35
    script.lock_and_run()
2014
36
2015
37
0
2016
=== modified file 'cronscripts/buildd-queue-builder.py'
2017
--- cronscripts/buildd-queue-builder.py	2009-06-24 20:52:01 +0000
2018
+++ cronscripts/buildd-queue-builder.py	2010-05-05 01:58:48 +0000
2019
@@ -1,4 +1,4 @@
2021
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2022
2
#
2
#
2023
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2024
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2025
5
5
2026
=== modified file 'cronscripts/buildd-retry-depwait.py'
2027
--- cronscripts/buildd-retry-depwait.py	2009-06-24 20:52:01 +0000
2028
+++ cronscripts/buildd-retry-depwait.py	2010-05-05 01:58:48 +0000
2029
@@ -1,4 +1,4 @@
2031
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2032
2
#
2
#
2033
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2034
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2035
5
5
2036
=== modified file 'cronscripts/buildd-slave-scanner.py'
2037
--- cronscripts/buildd-slave-scanner.py	2009-06-24 20:52:01 +0000
2038
+++ cronscripts/buildd-slave-scanner.py	2010-05-05 01:58:48 +0000
2039
@@ -1,4 +1,4 @@
2041
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2042
2
#
2
#
2043
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2044
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2045
5
5
2046
=== added file 'cronscripts/calculate-bug-heat.py'
2047
--- cronscripts/calculate-bug-heat.py	1970-01-01 00:00:00 +0000
2048
+++ cronscripts/calculate-bug-heat.py	2010-05-05 01:58:48 +0000
2049
@@ -0,0 +1,33 @@
2050
1
#!/usr/bin/python2.5 -S
2051
2
#
2052
3
# Copyright 2010 Canonical Ltd.  This software is licensed under the
2053
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2054
5
2055
6
# pylint: disable-msg=W0403
2056
7
2057
8
"""Calculate bug heat."""
2058
9
2059
10
__metaclass__ = type
2060
11
2061
12
import _pythonpath
2062
13
2063
14
from canonical.launchpad.webapp import errorlog
2064
15
2065
16
from lp.services.job.runner import JobCronScript
2066
17
from lp.bugs.interfaces.bugjob import ICalculateBugHeatJobSource
2067
18
2068
19
2069
20
class RunCalculateBugHeat(JobCronScript):
2070
21
    """Run BranchScanJob jobs."""
2071
22
2072
23
    config_name = 'calculate_bug_heat'
2073
24
    source_interface = ICalculateBugHeatJobSource
2074
25
2075
26
    def main(self):
2076
27
        errorlog.globalErrorUtility.configure(self.config_name)
2077
28
        return super(RunCalculateBugHeat, self).main()
2078
29
2079
30
2080
31
if __name__ == '__main__':
2081
32
    script = RunCalculateBugHeat()
2082
33
    script.lock_and_run()
2083
0
34
2084
=== modified file 'cronscripts/check-teamparticipation.py'
2085
--- cronscripts/check-teamparticipation.py	2009-07-29 01:14:50 +0000
2086
+++ cronscripts/check-teamparticipation.py	2010-05-05 01:58:48 +0000
2087
@@ -1,4 +1,4 @@
2089
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2090
2
#
2
#
2091
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2092
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2093
@@ -18,11 +18,11 @@
2094
18
situation, but that's not a simple thing and this should do for now.
18
situation, but that's not a simple thing and this should do for now.
2095
19
"""
19
"""
2096
20
20
2097
21
import _pythonpath
2098
22
2099
21
import optparse
23
import optparse
2100
22
import sys
24
import sys
2101
23
25
2102
24
import _pythonpath
2103
25
2104
26
from canonical.database.sqlbase import cursor
26
from canonical.database.sqlbase import cursor
2105
27
from canonical.launchpad.scripts import (
27
from canonical.launchpad.scripts import (
2106
28
    execute_zcml_for_scripts, logger_options, logger)
28
    execute_zcml_for_scripts, logger_options, logger)
2107
29
29
2108
=== modified file 'cronscripts/checkwatches.py'
2109
--- cronscripts/checkwatches.py	2009-10-05 13:08:10 +0000
2110
+++ cronscripts/checkwatches.py	2010-05-05 01:58:48 +0000
2111
@@ -1,4 +1,4 @@
2113
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2114
2
#
2
#
2115
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2116
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2117
5
5
2118
=== modified file 'cronscripts/code-import-dispatcher.py'
2119
--- cronscripts/code-import-dispatcher.py	2009-06-24 20:52:01 +0000
2120
+++ cronscripts/code-import-dispatcher.py	2010-05-05 01:58:48 +0000
2121
@@ -1,4 +1,4 @@
2123
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2124
2
#
2
#
2125
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2126
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2127
@@ -18,6 +18,12 @@
2128
18
18
2129
19
class CodeImportDispatcherScript(LaunchpadScript):
19
class CodeImportDispatcherScript(LaunchpadScript):
2130
20
20
2131
21
    def add_my_options(self):
2132
22
        self.parser.add_option(
2133
23
            "--max-jobs", dest="max_jobs", type=int,
2134
24
            default=config.codeimportdispatcher.max_jobs_per_machine,
2135
25
            help="The maximum number of jobs to run on this machine.")
2136
26
2137
21
    def run(self, use_web_security=False, implicit_begin=True,
27
    def run(self, use_web_security=False, implicit_begin=True,
2138
22
            isolation=None):
28
            isolation=None):
2139
23
        """See `LaunchpadScript.run`.
29
        """See `LaunchpadScript.run`.
2140
@@ -30,7 +36,8 @@
2141
30
    def main(self):
36
    def main(self):
2142
31
        globalErrorUtility.configure('codeimportdispatcher')
37
        globalErrorUtility.configure('codeimportdispatcher')
2143
32
38
2145
33
        CodeImportDispatcher(self.logger).findAndDispatchJob(
39
        dispatcher = CodeImportDispatcher(self.logger, self.options.max_jobs)
2146
40
        dispatcher.findAndDispatchJobs(
2147
34
            ServerProxy(config.codeimportdispatcher.codeimportscheduler_url))
41
            ServerProxy(config.codeimportdispatcher.codeimportscheduler_url))
2148
35
42
2149
36
43
2150
37
44
2151
=== modified file 'cronscripts/create-debwatches.py'
2152
--- cronscripts/create-debwatches.py	2009-06-24 20:52:01 +0000
2153
+++ cronscripts/create-debwatches.py	2010-05-05 01:58:48 +0000
2154
@@ -1,4 +1,4 @@
2156
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2157
2
#
2
#
2158
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2159
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2160
@@ -11,9 +11,9 @@
2161
11
11
2162
12
__metaclass__ = type
12
__metaclass__ = type
2163
13
13
2164
14
import _pythonpath
2165
14
import os
15
import os
2166
15
import logging
16
import logging
2167
16
import _pythonpath
2168
17
17
2169
18
# zope bits
18
# zope bits
2170
19
from zope.component import getUtility
19
from zope.component import getUtility
2171
20
20
2172
=== modified file 'cronscripts/create_merge_proposals.py'
2173
--- cronscripts/create_merge_proposals.py	2009-09-03 19:46:42 +0000
2174
+++ cronscripts/create_merge_proposals.py	2010-05-05 01:58:48 +0000
2175
@@ -1,4 +1,4 @@
2177
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2178
2
#
2
#
2179
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2180
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2181
5
5
2182
=== modified file 'cronscripts/distributionmirror-prober.py'
2183
--- cronscripts/distributionmirror-prober.py	2009-06-24 20:52:01 +0000
2184
+++ cronscripts/distributionmirror-prober.py	2010-05-05 01:58:48 +0000
2185
@@ -1,4 +1,4 @@
2187
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2188
2
#
2
#
2189
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2190
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2191
5
5
2192
=== renamed file 'cronscripts/expire-ppa-binaries.py' => 'cronscripts/expire-archive-files.py'
2193
--- cronscripts/expire-ppa-binaries.py	2009-06-24 20:52:01 +0000
2194
+++ cronscripts/expire-archive-files.py	2010-05-05 01:58:48 +0000
2195
@@ -1,4 +1,4 @@
2197
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2198
2
#
2
#
2199
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2200
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2201
@@ -12,11 +12,11 @@
2202
12
import _pythonpath
12
import _pythonpath
2203
13
13
2204
14
from canonical.config import config
14
from canonical.config import config
2206
15
from lp.soyuz.scripts.expire_ppa_binaries import PPABinaryExpirer
15
from lp.soyuz.scripts.expire_archive_files import ArchiveExpirer
2207
16
16
2208
17
17
2209
18
if __name__ == '__main__':
18
if __name__ == '__main__':
2212
19
    script = PPABinaryExpirer(
19
    script = ArchiveExpirer(
2213
20
        'expire-ppa-binaries', dbuser=config.binaryfile_expire.dbuser)
20
        'expire-archive-files', dbuser=config.binaryfile_expire.dbuser)
2214
21
    script.lock_and_run()
21
    script.lock_and_run()
2215
22
22
2216
23
23
2217
=== modified file 'cronscripts/expire-bugtasks.py'
2218
--- cronscripts/expire-bugtasks.py	2009-06-24 20:52:01 +0000
2219
+++ cronscripts/expire-bugtasks.py	2010-05-05 01:58:48 +0000
2220
@@ -1,4 +1,4 @@
2222
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2223
2
#
2
#
2224
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2225
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2226
5
5
2227
=== modified file 'cronscripts/expire-questions.py'
2228
--- cronscripts/expire-questions.py	2009-06-24 20:52:01 +0000
2229
+++ cronscripts/expire-questions.py	2010-05-05 01:58:48 +0000
2230
@@ -1,4 +1,4 @@
2232
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2233
2
#
2
#
2234
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2235
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2236
5
5
2237
=== modified file 'cronscripts/flag-expired-memberships.py'
2238
--- cronscripts/flag-expired-memberships.py	2009-06-24 20:52:01 +0000
2239
+++ cronscripts/flag-expired-memberships.py	2010-05-05 01:58:48 +0000
2240
@@ -1,4 +1,4 @@
2242
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2243
2
#
2
#
2244
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2245
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2246
5
5
2247
=== modified file 'cronscripts/foaf-update-karma-cache.py'
2248
--- cronscripts/foaf-update-karma-cache.py	2009-06-24 20:52:01 +0000
2249
+++ cronscripts/foaf-update-karma-cache.py	2010-05-05 01:58:48 +0000
2250
@@ -1,4 +1,4 @@
2252
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2253
2
#
2
#
2254
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2255
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2256
@@ -10,7 +10,8 @@
2257
10
from zope.component import getUtility
10
from zope.component import getUtility
2258
11
11
2259
12
from canonical.config import config
12
from canonical.config import config
2261
13
from canonical.database.sqlbase import ISOLATION_LEVEL_AUTOCOMMIT
13
from canonical.database.sqlbase import (
2262
14
    ISOLATION_LEVEL_AUTOCOMMIT, flush_database_updates)
2263
14
from canonical.launchpad.interfaces import IKarmaCacheManager, NotFoundError
15
from canonical.launchpad.interfaces import IKarmaCacheManager, NotFoundError
2264
15
from lp.services.scripts.base import LaunchpadCronScript
16
from lp.services.scripts.base import LaunchpadCronScript
2265
16
17
2266
@@ -83,6 +84,7 @@
2267
83
        scaling = self.calculate_scaling(results)
84
        scaling = self.calculate_scaling(results)
2268
84
        for entry in results:
85
        for entry in results:
2269
85
            self.update_one_karma_cache_entry(entry, scaling)
86
            self.update_one_karma_cache_entry(entry, scaling)
2270
87
        flush_database_updates()
2271
86
88
2272
87
        # Delete the entries we're going to replace.
89
        # Delete the entries we're going to replace.
2273
88
        self.cur.execute("DELETE FROM KarmaCache WHERE category IS NULL")
90
        self.cur.execute("DELETE FROM KarmaCache WHERE category IS NULL")
2274
@@ -121,7 +123,7 @@
2275
121
        # VACUUM KarmaTotalCache since we have just touched every row in it.
123
        # VACUUM KarmaTotalCache since we have just touched every row in it.
2276
122
        self.cur.execute("""VACUUM KarmaTotalCache""")
124
        self.cur.execute("""VACUUM KarmaTotalCache""")
2277
123
125
2279
124
        # Insert new records into the KarmaTotalCache table. 
126
        # Insert new records into the KarmaTotalCache table.
2280
125
127
2281
126
        # XXX: salgado 2007-02-06:
128
        # XXX: salgado 2007-02-06:
2282
127
        # If deadlocks ever become a problem, first LOCK the
129
        # If deadlocks ever become a problem, first LOCK the
2283
@@ -146,7 +148,7 @@
2284
146
148
2285
147
    def C_add_karmacache_sums(self):
149
    def C_add_karmacache_sums(self):
2286
148
        self.logger.info("Step C: Calculating KarmaCache sums")
150
        self.logger.info("Step C: Calculating KarmaCache sums")
2288
149
        # We must issue some SUM queries to insert the karma totals for: 
151
        # We must issue some SUM queries to insert the karma totals for:
2289
150
        # - All actions of a person on a given product.
152
        # - All actions of a person on a given product.
2290
151
        # - All actions of a person on a given distribution.
153
        # - All actions of a person on a given distribution.
2291
152
        # - All actions of a person on a given project.
154
        # - All actions of a person on a given project.
2292
@@ -156,7 +158,7 @@
2293
156
158
2294
157
        # - All actions with a specific category of a person.
159
        # - All actions with a specific category of a person.
2295
158
        self.cur.execute("""
160
        self.cur.execute("""
2297
159
            INSERT INTO KarmaCache 
161
            INSERT INTO KarmaCache
2298
160
                (person, category, karmavalue, product, distribution,
162
                (person, category, karmavalue, product, distribution,
2299
161
                 sourcepackagename, project)
163
                 sourcepackagename, project)
2300
162
            SELECT person, category, SUM(karmavalue), NULL, NULL, NULL, NULL
164
            SELECT person, category, SUM(karmavalue), NULL, NULL, NULL, NULL
2301
@@ -167,7 +169,7 @@
2302
167
169
2303
168
        # - All actions of a person on a given product.
170
        # - All actions of a person on a given product.
2304
169
        self.cur.execute("""
171
        self.cur.execute("""
2306
170
            INSERT INTO KarmaCache 
172
            INSERT INTO KarmaCache
2307
171
                (person, category, karmavalue, product, distribution,
173
                (person, category, karmavalue, product, distribution,
2308
172
                 sourcepackagename, project)
174
                 sourcepackagename, project)
2309
173
            SELECT person, NULL, SUM(karmavalue), product, NULL, NULL, NULL
175
            SELECT person, NULL, SUM(karmavalue), product, NULL, NULL, NULL
2310
@@ -178,7 +180,7 @@
2311
178
180
2312
179
        # - All actions of a person on a given distribution.
181
        # - All actions of a person on a given distribution.
2313
180
        self.cur.execute("""
182
        self.cur.execute("""
2315
181
            INSERT INTO KarmaCache 
183
            INSERT INTO KarmaCache
2316
182
                (person, category, karmavalue, product, distribution,
184
                (person, category, karmavalue, product, distribution,
2317
183
                 sourcepackagename, project)
185
                 sourcepackagename, project)
2318
184
            SELECT person, NULL, SUM(karmavalue), NULL, distribution, NULL, NULL
186
            SELECT person, NULL, SUM(karmavalue), NULL, distribution, NULL, NULL
2319
@@ -189,7 +191,7 @@
2320
189
191
2321
190
        # - All actions of a person on a given project.
192
        # - All actions of a person on a given project.
2322
191
        self.cur.execute("""
193
        self.cur.execute("""
2324
192
            INSERT INTO KarmaCache 
194
            INSERT INTO KarmaCache
2325
193
                (person, category, karmavalue, product, distribution,
195
                (person, category, karmavalue, product, distribution,
2326
194
                 sourcepackagename, project)
196
                 sourcepackagename, project)
2327
195
            SELECT person, NULL, SUM(karmavalue), NULL, NULL, NULL,
197
            SELECT person, NULL, SUM(karmavalue), NULL, NULL, NULL,
2328
@@ -206,7 +208,7 @@
2329
206
        # inserted here will be included in the calculation of the overall
208
        # inserted here will be included in the calculation of the overall
2330
207
        # karma of a person on a given project.
209
        # karma of a person on a given project.
2331
208
        self.cur.execute("""
210
        self.cur.execute("""
2333
209
            INSERT INTO KarmaCache 
211
            INSERT INTO KarmaCache
2334
210
                (person, category, karmavalue, product, distribution,
212
                (person, category, karmavalue, product, distribution,
2335
211
                 sourcepackagename, project)
213
                 sourcepackagename, project)
2336
212
            SELECT person, category, SUM(karmavalue), NULL, NULL, NULL,
214
            SELECT person, category, SUM(karmavalue), NULL, NULL, NULL,
2337
@@ -244,15 +246,18 @@
2338
244
                scaling[category] = 1
246
                scaling[category] = 1
2339
245
            else:
247
            else:
2340
246
                scaling[category] = float(largest_total) / float(points)
248
                scaling[category] = float(largest_total) / float(points)
2341
247
            self.logger.debug('Scaling %s by a factor of %0.4f'
2342
248
                              % (categories[category], scaling[category]))
2343
249
            max_scaling = config.karmacacheupdater.max_scaling
249
            max_scaling = config.karmacacheupdater.max_scaling
2344
250
            if scaling[category] > max_scaling:
250
            if scaling[category] > max_scaling:
2345
251
                self.logger.info(
2346
252
                    'Scaling %s by a factor of %0.4f (capped to %0.4f)'
2347
253
                    % (categories[category], scaling[category], max_scaling))
2348
251
                scaling[category] = max_scaling
254
                scaling[category] = max_scaling
2351
252
                self.logger.debug('Reducing %s scaling to %d to avoid spikes' 
255
            else:
2352
253
                                  % (categories[category], max_scaling))
256
                self.logger.info(
2353
257
                    'Scaling %s by a factor of %0.4f'
2354
258
                    % (categories[category], scaling[category]))
2355
254
        return scaling
259
        return scaling
2357
255
    
260
2358
256
    def update_one_karma_cache_entry(self, entry, scaling):
261
    def update_one_karma_cache_entry(self, entry, scaling):
2359
257
        """Updates an individual (non-summed) KarmaCache entry.
262
        """Updates an individual (non-summed) KarmaCache entry.
2360
258
263
2361
@@ -262,7 +267,7 @@
2362
262
        """
267
        """
2363
263
        (person_id, category_id, product_id, distribution_id, points) = entry
268
        (person_id, category_id, product_id, distribution_id, points) = entry
2364
264
        points *= scaling[category_id] # Scaled. wow.
269
        points *= scaling[category_id] # Scaled. wow.
2366
265
        self.logger.debug("Setting person_id=%d, category_id=%d, points=%d" 
270
        self.logger.debug("Setting person_id=%d, category_id=%d, points=%d"
2367
266
                          % (person_id, category_id, points))
271
                          % (person_id, category_id, points))
2368
267
272
2369
268
        points = int(points)
273
        points = int(points)
2370
@@ -285,7 +290,7 @@
2371
285
290
2372
286
291
2373
287
if __name__ == '__main__':
292
if __name__ == '__main__':
2375
288
    script = KarmaCacheUpdater('karma-update', 
293
    script = KarmaCacheUpdater('karma-update',
2376
289
        dbuser=config.karmacacheupdater.dbuser)
294
        dbuser=config.karmacacheupdater.dbuser)
2377
290
    script.lock_and_run(implicit_begin=True)
295
    script.lock_and_run(implicit_begin=True)
2378
291
296
2379
292
297
2380
=== modified file 'cronscripts/garbo-daily.py'
2381
--- cronscripts/garbo-daily.py	2009-06-24 20:52:01 +0000
2382
+++ cronscripts/garbo-daily.py	2010-05-05 01:58:48 +0000
2383
@@ -1,4 +1,4 @@
2385
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2386
2
#
2
#
2387
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2388
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2389
5
5
2390
=== modified file 'cronscripts/garbo-hourly.py'
2391
--- cronscripts/garbo-hourly.py	2009-06-24 20:52:01 +0000
2392
+++ cronscripts/garbo-hourly.py	2010-05-05 01:58:48 +0000
2393
@@ -1,4 +1,4 @@
2395
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2396
2
#
2
#
2397
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2398
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2399
5
5
2400
=== modified file 'cronscripts/generate-ppa-htaccess.py'
2401
--- cronscripts/generate-ppa-htaccess.py	2009-06-24 20:52:01 +0000
2402
+++ cronscripts/generate-ppa-htaccess.py	2010-05-05 01:58:48 +0000
2403
@@ -1,4 +1,4 @@
2405
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2406
2
#
2
#
2407
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2408
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2409
5
5
2410
=== modified file 'cronscripts/language-pack-exporter.py'
2411
--- cronscripts/language-pack-exporter.py	2009-07-17 00:26:05 +0000
2412
+++ cronscripts/language-pack-exporter.py	2010-05-05 01:58:48 +0000
2413
@@ -1,4 +1,4 @@
2415
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2416
2
#
2
#
2417
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2418
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2419
5
5
2420
=== modified file 'cronscripts/librarian-gc.py'
2421
--- cronscripts/librarian-gc.py	2009-06-24 20:52:01 +0000
2422
+++ cronscripts/librarian-gc.py	2010-05-05 01:58:48 +0000
2423
@@ -1,4 +1,4 @@
2425
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2426
2
#
2
#
2427
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2428
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2429
@@ -46,16 +46,17 @@
2430
46
                help="Skip removing expired TemporaryBlobStorage rows"
46
                help="Skip removing expired TemporaryBlobStorage rows"
2431
47
                )
47
                )
2432
48
        self.parser.add_option(
48
        self.parser.add_option(
2433
49
                '', "--skip-expired", action="store_true", default=False,
2434
50
                dest="skip_expired",
2435
51
                help="Skip flagging expired files for deletion."
2436
52
                )
2437
53
        self.parser.add_option(
2438
54
                '', "--skip-files", action="store_true", default=False,
49
                '', "--skip-files", action="store_true", default=False,
2439
55
                dest="skip_files",
50
                dest="skip_files",
2440
56
                help="Skip removing files on disk with no database references"
51
                help="Skip removing files on disk with no database references"
2441
57
                     " or flagged for deletion."
52
                     " or flagged for deletion."
2442
58
                )
53
                )
2443
54
        self.parser.add_option(
2444
55
                '', "--skip-expiry", action="store_true", default=False,
2445
56
                dest="skip_expiry",
2446
57
                help="Skip expiring aliases with an expiry date in the past."
2447
58
                )
2448
59
2449
59
60
2450
60
    def main(self):
61
    def main(self):
2451
61
        librariangc.log = self.logger
62
        librariangc.log = self.logger
2452
@@ -71,6 +72,8 @@
2453
71
72
2454
72
        # Note that each of these next steps will issue commit commands
73
        # Note that each of these next steps will issue commit commands
2455
73
        # as appropriate to make this script transaction friendly
74
        # as appropriate to make this script transaction friendly
2456
75
        if not self.options.skip_expiry:
2457
76
            librariangc.expire_aliases(conn)
2458
74
        if not self.options.skip_content:
77
        if not self.options.skip_content:
2459
75
            librariangc.delete_unreferenced_content(conn) # first sweep
78
            librariangc.delete_unreferenced_content(conn) # first sweep
2460
76
        if not self.options.skip_blobs:
79
        if not self.options.skip_blobs:
2461
@@ -81,8 +84,6 @@
2462
81
            librariangc.delete_unreferenced_aliases(conn)
84
            librariangc.delete_unreferenced_aliases(conn)
2463
82
        if not self.options.skip_content:
85
        if not self.options.skip_content:
2464
83
            librariangc.delete_unreferenced_content(conn) # second sweep
86
            librariangc.delete_unreferenced_content(conn) # second sweep
2465
84
        if not self.options.skip_expired:
2466
85
            librariangc.flag_expired_files(conn)
2467
86
        if not self.options.skip_files:
87
        if not self.options.skip_files:
2468
87
            librariangc.delete_unwanted_files(conn)
88
            librariangc.delete_unwanted_files(conn)
2469
88
89
2470
89
90
2471
=== renamed file 'cronscripts/mpcreationjobs.py' => 'cronscripts/merge-proposal-jobs.py'
2472
--- cronscripts/mpcreationjobs.py	2009-09-03 19:04:28 +0000
2473
+++ cronscripts/merge-proposal-jobs.py	2010-05-05 01:58:48 +0000
2474
@@ -1,48 +1,40 @@
2476
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2477
2
#
2
#
2479
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009, 2010 Canonical Ltd.  This software is licensed under the
2480
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2481
5
5
2482
6
# pylint: disable-msg=W0403
6
# pylint: disable-msg=W0403
2483
7
7
2485
8
"""Handle new BranchMergeProposals.
8
"""Handle jobs for BranchMergeProposals.
2486
9
9
2489
10
This script generates a diff for the merge proposal if needed, then notifies
10
This script handles all job types for branch merge proposals.
2488
11
all interested parties about the merge proposal.
2490
12
"""
11
"""
2491
13
12
2492
14
__metaclass__ = type
13
__metaclass__ = type
2493
15
14
2494
16
import _pythonpath
15
import _pythonpath
2495
17
from zope.component import getUtility
2496
18
16
2500
19
from canonical.config import config
17
# The following line is a horrible hack, but unfortunately necessary right now
2501
20
from lp.codehosting.vfs import get_scanner_server
18
# to stop import errors from circular imports.
2502
21
from lp.services.job.runner import JobRunner
19
import canonical.launchpad.interfaces
2503
22
from lp.code.interfaces.branchmergeproposal import (
20
from lp.code.interfaces.branchmergeproposal import (
2524
23
    IMergeProposalCreatedJobSource,)
21
    IBranchMergeProposalJobSource,
2525
24
from lp.services.scripts.base import LaunchpadCronScript
22
    )
2526
25
from canonical.launchpad.webapp.errorlog import globalErrorUtility
23
from lp.services.job.runner import JobCronScript, TwistedJobRunner
2527
26
24
2528
27
25
2529
28
class RunMergeProposalCreatedJobs(LaunchpadCronScript):
26
class RunMergeProposalJobs(JobCronScript):
2530
29
    """Run merge proposal creation jobs."""
27
    """Run all merge proposal jobs."""
2531
30
28
2532
31
    def main(self):
29
    config_name = 'merge_proposal_jobs'
2533
32
        globalErrorUtility.configure('mpcreationjobs')
30
    source_interface = IBranchMergeProposalJobSource
2534
33
        job_source = getUtility(IMergeProposalCreatedJobSource)
31
2535
34
        runner = JobRunner.fromReady(job_source, self.logger)
32
    def __init__(self):
2536
35
        server = get_scanner_server()
33
        super(RunMergeProposalJobs, self).__init__(
2537
36
        server.setUp()
34
            runner_class=TwistedJobRunner,
2538
37
        try:
35
            script_name='merge-proposal-jobs')
2519
38
            runner.runAll()
2520
39
        finally:
2521
40
            server.tearDown()
2522
41
        self.logger.info(
2523
42
            'Ran %d MergeProposalCreatedJobs.', len(runner.completed_jobs))
2539
43
36
2540
44
37
2541
45
if __name__ == '__main__':
38
if __name__ == '__main__':
2544
46
    script = RunMergeProposalCreatedJobs(
39
    script = RunMergeProposalJobs()
2543
47
        'mpcreationjobs', config.mpcreationjobs.dbuser)
2545
48
    script.lock_and_run()
40
    script.lock_and_run()
2546
49
41
2547
=== modified file 'cronscripts/mirror-prober.sh'
2548
--- cronscripts/mirror-prober.sh	2009-06-24 20:52:01 +0000
2549
+++ cronscripts/mirror-prober.sh	2010-05-05 01:58:48 +0000
2550
@@ -39,10 +39,10 @@
2551
39
cd /srv/launchpad.net/production/launchpad/cronscripts
39
cd /srv/launchpad.net/production/launchpad/cronscripts
2552
40
40
2553
41
echo '== Distribution mirror prober (archive)' `date` ==
41
echo '== Distribution mirror prober (archive)' `date` ==
2555
42
python2.4 distributionmirror-prober.py --content-type=archive --max-mirrors=20
42
python2.5 -S distributionmirror-prober.py --content-type=archive --max-mirrors=20
2556
43
43
2557
44
echo '== Distribution mirror prober (cdimage)' `date` ==
44
echo '== Distribution mirror prober (cdimage)' `date` ==
2559
45
python2.4 distributionmirror-prober.py --content-type=cdimage --max-mirrors=30
45
python2.5 -S distributionmirror-prober.py --content-type=cdimage --max-mirrors=30
2560
46
46
2561
47
rm -f $LOCK
47
rm -f $LOCK
2562
48
48
2563
49
49
2564
=== modified file 'cronscripts/nightly.sh'
2565
--- cronscripts/nightly.sh	2009-08-10 16:57:14 +0000
2566
+++ cronscripts/nightly.sh	2010-05-05 01:58:48 +0000
2567
@@ -3,11 +3,11 @@
2568
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2569
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2570
5
5
2572
6
# This script performs nightly chores. It should be run from 
6
# This script performs nightly chores. It should be run from
2573
7
# cron as the launchpad user once a day. Typically the output
7
# cron as the launchpad user once a day. Typically the output
2574
8
# will be sent to an email address for inspection.
8
# will be sent to an email address for inspection.
2575
9
9
2577
10
# Note that http/ftp proxies are needed by the product 
10
# Note that http/ftp proxies are needed by the product
2578
11
# release finder
11
# release finder
2579
12
12
2580
13
# Only run this script on loganberry
13
# Only run this script on loganberry
2581
@@ -42,41 +42,41 @@
2582
42
cd /srv/launchpad.net/production/launchpad/cronscripts
42
cd /srv/launchpad.net/production/launchpad/cronscripts
2583
43
43
2584
44
echo == Expiring memberships `date` ==
44
echo == Expiring memberships `date` ==
2586
45
python2.4 flag-expired-memberships.py -q
45
python2.5 -S flag-expired-memberships.py -q
2587
46
46
2588
47
echo == Allocating revision karma `date` ==
47
echo == Allocating revision karma `date` ==
2590
48
python2.4 allocate-revision-karma.py -q
48
python2.5 -S allocate-revision-karma.py -q
2591
49
49
2592
50
echo == Recalculating karma `date` ==
50
echo == Recalculating karma `date` ==
2594
51
python2.4 foaf-update-karma-cache.py -q
51
python2.5 -S foaf-update-karma-cache.py -q
2595
52
52
2596
53
echo == Updating cached statistics `date` ==
53
echo == Updating cached statistics `date` ==
2598
54
python2.4 update-stats.py -q
54
python2.5 -S update-stats.py -q
2599
55
55
2600
56
echo == Expiring questions `date` ==
56
echo == Expiring questions `date` ==
2602
57
python2.4 expire-questions.py
57
python2.5 -S expire-questions.py
2603
58
58
2604
59
### echo == Expiring bugs `date` ==
59
### echo == Expiring bugs `date` ==
2606
60
### python2.4 expire-bugtasks.py
60
### python2.5 -S expire-bugtasks.py
2607
61
61
2608
62
# checkwatches.py is scheduled in the /code/pqm/launchpad_crontabs branch.
62
# checkwatches.py is scheduled in the /code/pqm/launchpad_crontabs branch.
2609
63
### echo == Updating bug watches `date` ==
63
### echo == Updating bug watches `date` ==
2611
64
### python2.4 checkwatches.py
64
### python2.5 -S checkwatches.py
2612
65
65
2613
66
echo == Updating bugtask target name caches `date` ==
66
echo == Updating bugtask target name caches `date` ==
2615
67
python2.4 update-bugtask-targetnamecaches.py -q
67
python2.5 -S update-bugtask-targetnamecaches.py -q
2616
68
68
2617
69
echo == Updating personal standings `date` ==
69
echo == Updating personal standings `date` ==
2619
70
python2.4 update-standing.py -q
70
python2.5 -S update-standing.py -q
2620
71
71
2621
72
echo == Updating CVE database `date` ==
72
echo == Updating CVE database `date` ==
2623
73
python2.4 update-cve.py -q
73
python2.5 -S update-cve.py -q
2624
74
74
2625
75
echo == Updating package cache `date` ==
75
echo == Updating package cache `date` ==
2627
76
python2.4 update-pkgcache.py -q
76
python2.5 -S update-pkgcache.py -q
2628
77
77
2629
78
echo == Product Release Finder `date` ==
78
echo == Product Release Finder `date` ==
2631
79
python2.4 product-release-finder.py -q
79
python2.5 -S product-release-finder.py -q
2632
80
80
2633
81
81
2634
82
rm -f $LOCK
82
rm -f $LOCK
2635
83
83
2636
=== modified file 'cronscripts/oops-prune.py'
2637
--- cronscripts/oops-prune.py	2009-06-24 20:52:01 +0000
2638
+++ cronscripts/oops-prune.py	2010-05-05 01:58:48 +0000
2639
@@ -1,4 +1,4 @@
2641
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2642
2
#
2
#
2643
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2644
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2645
5
5
2646
=== modified file 'cronscripts/parse-librarian-apache-access-logs.py'
2647
--- cronscripts/parse-librarian-apache-access-logs.py	2009-09-11 12:11:04 +0000
2648
+++ cronscripts/parse-librarian-apache-access-logs.py	2010-05-05 01:58:48 +0000
2649
@@ -1,4 +1,4 @@
2651
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2652
2
#
2
#
2653
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2654
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2655
5
5
2656
=== added file 'cronscripts/parse-ppa-apache-access-logs.py'
2657
--- cronscripts/parse-ppa-apache-access-logs.py	1970-01-01 00:00:00 +0000
2658
+++ cronscripts/parse-ppa-apache-access-logs.py	2010-05-05 01:58:48 +0000
2659
@@ -0,0 +1,61 @@
2660
1
#!/usr/bin/python2.5 -S
2661
2
#
2662
3
# Copyright 2010 Canonical Ltd.  This software is licensed under the
2663
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2664
5
2665
6
"""Parse PPA apache logs to find out download counts for each file."""
2666
7
2667
8
__metaclass__ = type
2668
9
2669
10
import _pythonpath
2670
11
2671
12
import functools
2672
13
2673
14
from zope.component import getUtility
2674
15
2675
16
from canonical.config import config
2676
17
# XXX: wgrant 2010-03-16 bug=539496: Importing directly from
2677
18
# lp.registry.interfaces.person results in a circular import.
2678
19
from canonical.launchpad.interfaces import IPersonSet
2679
20
from lp.soyuz.interfaces.archive import NoSuchPPA
2680
21
from lp.soyuz.scripts.ppa_apache_log_parser import DBUSER, get_ppa_file_key
2681
22
from lp.services.apachelogparser.script import ParseApacheLogs
2682
23
2683
24
2684
25
class ParsePPAApacheLogs(ParseApacheLogs):
2685
26
    """An Apache log parser for PPA downloads."""
2686
27
2687
28
    def setUpUtilities(self):
2688
29
        """See `ParseApacheLogs`."""
2689
30
        self.person_set = getUtility(IPersonSet)
2690
31
2691
32
    @property
2692
33
    def root(self):
2693
34
        """See `ParseApacheLogs`."""
2694
35
        return config.ppa_apache_log_parser.logs_root
2695
36
2696
37
    def getDownloadKey(self, path):
2697
38
        """See `ParseApacheLogs`."""
2698
39
        return get_ppa_file_key(path)
2699
40
2700
41
    def getDownloadCountUpdater(self, file_id):
2701
42
        """See `ParseApacheLogs`."""
2702
43
        person = self.person_set.getByName(file_id[0])
2703
44
        if person is None:
2704
45
            return
2705
46
        try:
2706
47
            archive = person.getPPAByName(file_id[1])
2707
48
        except NoSuchPPA:
2708
49
            return None
2709
50
        # file_id[2] (distro) isn't used yet, since getPPAByName
2710
51
        # hardcodes Ubuntu.
2711
52
        bpr = archive.getBinaryPackageReleaseByFileName(file_id[3])
2712
53
        if bpr is None:
2713
54
            return None
2714
55
2715
56
        return functools.partial(archive.updatePackageDownloadCount, bpr)
2716
57
2717
58
2718
59
if __name__ == '__main__':
2719
60
    script = ParsePPAApacheLogs('parse-ppa-apache-logs', DBUSER)
2720
61
    script.lock_and_run()
2721
0
62
2722
=== modified file 'cronscripts/ppa-generate-keys.py'
2723
--- cronscripts/ppa-generate-keys.py	2009-06-24 20:52:01 +0000
2724
+++ cronscripts/ppa-generate-keys.py	2010-05-05 01:58:48 +0000
2725
@@ -1,4 +1,4 @@
2727
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2728
2
#
2
#
2729
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2730
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2731
5
5
2732
=== added file 'cronscripts/process-apport-blobs.py'
2733
--- cronscripts/process-apport-blobs.py	1970-01-01 00:00:00 +0000
2734
+++ cronscripts/process-apport-blobs.py	2010-05-05 01:58:48 +0000
2735
@@ -0,0 +1,33 @@
2736
1
#!/usr/bin/python2.5 -S
2737
2
#
2738
3
# Copyright 2010 Canonical Ltd.  This software is licensed under the
2739
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2740
5
2741
6
# pylint: disable-msg=W0403
2742
7
2743
8
"""Process uploaded Apport BLOBs."""
2744
9
2745
10
__metaclass__ = type
2746
11
2747
12
import _pythonpath
2748
13
2749
14
from canonical.launchpad.webapp import errorlog
2750
15
2751
16
from lp.services.job.runner import JobCronScript
2752
17
from lp.bugs.interfaces.apportjob import IProcessApportBlobJobSource
2753
18
2754
19
2755
20
class RunProcessApportBlobs(JobCronScript):
2756
21
    """Run ProcessApportBlobJobs."""
2757
22
2758
23
    config_name = 'process_apport_blobs'
2759
24
    source_interface = IProcessApportBlobJobSource
2760
25
2761
26
    def main(self):
2762
27
        errorlog.globalErrorUtility.configure(self.config_name)
2763
28
        return super(RunProcessApportBlobs, self).main()
2764
29
2765
30
2766
31
if __name__ == '__main__':
2767
32
    script = RunProcessApportBlobs()
2768
33
    script.lock_and_run()
2769
0
34
2770
=== modified file 'cronscripts/process-hwdb-submissions.py'
2771
--- cronscripts/process-hwdb-submissions.py	2009-06-24 20:52:01 +0000
2772
+++ cronscripts/process-hwdb-submissions.py	2010-05-05 01:58:48 +0000
2773
@@ -1,4 +1,4 @@
2775
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2776
2
#
2
#
2777
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2778
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2779
@@ -24,7 +24,7 @@
2780
24
import _pythonpath
24
import _pythonpath
2781
25
25
2782
26
from lp.services.scripts.base import LaunchpadCronScript
26
from lp.services.scripts.base import LaunchpadCronScript
2784
27
from canonical.launchpad.scripts.hwdbsubmissions import (
27
from lp.hardwaredb.scripts.hwdbsubmissions import (
2785
28
    process_pending_submissions)
28
    process_pending_submissions)
2786
29
29
2787
30
30
2788
31
31
2789
=== modified file 'cronscripts/process-mail.py'
2790
--- cronscripts/process-mail.py	2009-06-24 20:52:01 +0000
2791
+++ cronscripts/process-mail.py	2010-05-05 01:58:48 +0000
2792
@@ -1,4 +1,4 @@
2794
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2795
2
#
2
#
2796
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2797
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2798
5
5
2799
=== modified file 'cronscripts/process-pending-packagediffs.py'
2800
--- cronscripts/process-pending-packagediffs.py	2009-06-24 20:52:01 +0000
2801
+++ cronscripts/process-pending-packagediffs.py	2010-05-05 01:58:48 +0000
2802
@@ -1,4 +1,4 @@
2804
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2805
2
#
2
#
2806
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2807
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2808
5
5
2809
=== modified file 'cronscripts/product-release-finder.py'
2810
--- cronscripts/product-release-finder.py	2009-06-24 20:52:01 +0000
2811
+++ cronscripts/product-release-finder.py	2010-05-05 01:58:48 +0000
2812
@@ -1,4 +1,4 @@
2814
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
2815
2
#
2
#
2816
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2817
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2818
5
5
2819
=== modified file 'cronscripts/publishing/cron.germinate'
2820
--- cronscripts/publishing/cron.germinate	2009-08-21 09:10:55 +0000
2821
+++ cronscripts/publishing/cron.germinate	2010-05-05 01:58:48 +0000
2822
@@ -11,6 +11,9 @@
2823
11
LOCKROOT=$ARCHIVEROOT/..
11
LOCKROOT=$ARCHIVEROOT/..
2824
12
GERMINATEROOT=$ARCHIVEROOT/../ubuntu-germinate
12
GERMINATEROOT=$ARCHIVEROOT/../ubuntu-germinate
2825
13
13
2826
14
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
2827
15
MAINTAINCE_CHECK=$LAUNCHPADROOT/cronscripts/publishing/maintenance-check.py
2828
16
2829
14
## Check to see if another germinate run is in progress
17
## Check to see if another germinate run is in progress
2830
15
18
2831
16
LOCKFILE=$LOCKROOT/cron.germinate.lock
19
LOCKFILE=$LOCKROOT/cron.germinate.lock
2832
@@ -32,7 +35,7 @@
2833
32
cd $GERMINATEROOT
35
cd $GERMINATEROOT
2834
33
36
2835
34
# Clean up temporary files
37
# Clean up temporary files
2837
35
rm -f germinate.output ALL ALL.sources UBUNTU-* KUBUNTU-* EDUBUNTU-* XUBUNTU-* MOBILE-* MYTHBUNTU-* UNR-*
38
rm -f germinate.output ALL ALL.sources UBUNTU-* KUBUNTU-* EDUBUNTU-* XUBUNTU-* MYTHBUNTU-* NETBOOK-*
2838
36
rm -f all_* all.sources_*
39
rm -f all_* all.sources_*
2839
37
40
2840
38
# Grab a local copy of Sources files
41
# Grab a local copy of Sources files
2841
@@ -41,11 +44,12 @@
2842
41
done
44
done
2843
42
45
2844
43
> "$MISCROOT/more-extra.override.$suite.main.new"
46
> "$MISCROOT/more-extra.override.$suite.main.new"
2846
44
for distro in ubuntu kubuntu edubuntu xubuntu mobile mythbuntu unr; do
47
2847
48
germinate_components=main,universe,restricted,multiverse
2848
49
for distro in ubuntu kubuntu edubuntu xubuntu mythbuntu netbook; do
2849
45
  DISTRO="$(echo $distro | tr a-z A-Z)"
50
  DISTRO="$(echo $distro | tr a-z A-Z)"
2850
46
  germinate_suite="$distro.$suite"
51
  germinate_suite="$distro.$suite"
2853
47
  germinate_components=main,universe,restricted,multiverse
52
  for arch in i386 amd64 powerpc sparc ia64 armel; do
2852
48
  for arch in i386 amd64 lpia powerpc sparc ia64 armel; do
2854
49
    # Grab local copy of Packages and InstallerPackages files
53
    # Grab local copy of Packages and InstallerPackages files
2855
50
    for component in main universe restricted multiverse; do
54
    for component in main universe restricted multiverse; do
2856
51
      zcat $ARCHIVEROOT/dists/"$suite"/"$component"/binary-$arch/Packages.gz > archive.ubuntu.com_"$suite"_"$component"_Packages
55
      zcat $ARCHIVEROOT/dists/"$suite"/"$component"/binary-$arch/Packages.gz > archive.ubuntu.com_"$suite"_"$component"_Packages
2857
@@ -122,4 +126,10 @@
2858
122
done
126
done
2859
123
echo " done."
127
echo " done."
2860
124
128
2861
129
# now generate the Supported extra overrides
2862
130
$MAINTAINCE_CHECK $suite > "$MISCROOT/more-extra.override.$suite.main.supported" 2> _maintenance-check.stderr
2863
131
if [ $? -eq 0 ]; then
2864
132
    cat "$MISCROOT/more-extra.override.$suite.main.supported" >> "$MISCROOT/more-extra.override.$suite.main.new"
2865
133
fi
2866
134
2867
125
mv -f "$MISCROOT/more-extra.override.$suite.main.new" "$MISCROOT/more-extra.override.$suite.main"
135
mv -f "$MISCROOT/more-extra.override.$suite.main.new" "$MISCROOT/more-extra.override.$suite.main"
2868
126
136
2869
=== added file 'cronscripts/publishing/cron.publish-copy-archives'
2870
--- cronscripts/publishing/cron.publish-copy-archives	1970-01-01 00:00:00 +0000
2871
+++ cronscripts/publishing/cron.publish-copy-archives	2010-05-05 01:58:48 +0000
2872
@@ -0,0 +1,83 @@
2873
1
#!/bin/sh
2874
2
#
2875
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2876
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2877
5
2878
6
# LPCONFIG will come from the environment so this script can run unaltered
2879
7
# on dogfood.
2880
8
if [ -z $LPCONFIG ]; then
2881
9
    echo LPCONFIG must be set to run this script.
2882
10
    exit 1
2883
11
fi
2884
12
2885
13
set -x
2886
14
set -e
2887
15
set -u
2888
16
2889
17
#
2890
18
# This script publishes the COPY (rebuild) archvies *only*.
2891
19
#
2892
20
2893
21
2894
22
# Informational -- this *MUST* match the database.
2895
23
ARCHIVEROOT=/srv/launchpad.net/ubuntu-archive/ubuntu
2896
24
DISTSROOT=$ARCHIVEROOT/dists
2897
25
OVERRIDEROOT=$ARCHIVEROOT/../ubuntu-overrides
2898
26
INDICES=$ARCHIVEROOT/indices
2899
27
PRODUCTION_CONFIG=ftpmaster
2900
28
2901
29
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
2902
30
    GNUPGHOME=/srv/launchpad.net/ubuntu-archive/gnupg-home
2903
31
else
2904
32
    echo GPG keys will come from ~/.gnupg
2905
33
    # GNUPGHOME does not need to be set, keys can come from ~/.gnupg.
2906
34
fi
2907
35
2908
36
# Configuration options.
2909
37
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
2910
38
LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
2911
39
DISTRONAME=ubuntu
2912
40
TRACEFILE=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
2913
41
2914
42
# Manipulate the environment.
2915
43
export GNUPGHOME
2916
44
PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools
2917
45
2918
46
# Claim the lock.
2919
47
if ! lockfile -r1 $LOCKFILE; then
2920
48
  echo "Could not claim lock file."
2921
49
  exit 1
2922
50
fi
2923
51
2924
52
# Lock claimed.
2925
53
2926
54
cleanup () {
2927
55
  echo "Cleaning up lockfile."
2928
56
  rm -f $LOCKFILE
2929
57
}
2930
58
2931
59
trap cleanup EXIT
2932
60
2933
61
# Process the accepted queue into the publishing records.
2934
62
process-accepted.py --copy-archive -v -v -v $DISTRONAME
2935
63
2936
64
# Publish the packages to disk.
2937
65
publish-distro.py -v -v --copy-archive -d $DISTRONAME
2938
66
2939
67
set +x
2940
68
2941
69
echo Removing uncompressed Packages and Sources files
2942
70
find ${DISTSROOT} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
2943
71
2944
72
# Copy in the indices.
2945
73
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
2946
74
    echo Copying the indices into place.
2947
75
    rm -f $INDICES/override.*
2948
76
    cp $OVERRIDEROOT/override.* $INDICES
2949
77
fi
2950
78
2951
79
# Timestamp our trace file to track when the last archive publisher run took
2952
80
# place.
2953
81
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
2954
82
    date -u > "$TRACEFILE"
2955
83
fi
2956
0
84
2957
=== modified file 'cronscripts/publishing/cron.publish-ftpmaster'
2958
--- cronscripts/publishing/cron.publish-ftpmaster	2009-06-24 20:52:01 +0000
2959
+++ cronscripts/publishing/cron.publish-ftpmaster	2010-05-05 01:58:48 +0000
2960
@@ -44,10 +44,13 @@
2961
44
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
44
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
2962
45
LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
45
LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
2963
46
DISTRONAME=ubuntu
46
DISTRONAME=ubuntu
2965
47
TOUCHLIST=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
47
TRACEFILE=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
2966
48
DSYNCLIST=$CACHEROOT/dsync.list
48
DSYNCLIST=$CACHEROOT/dsync.list
2967
49
MD5LIST=$INDICES/md5sums.gz
49
MD5LIST=$INDICES/md5sums.gz
2968
50
50
2969
51
# Mirrors to push at the end of the publishing run.
2970
52
MASTERMIRRORS="syowa frei scandium"
2971
53
2972
51
# Manipulate the environment.
54
# Manipulate the environment.
2973
52
export GNUPGHOME
55
export GNUPGHOME
2974
53
PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools
56
PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools
2975
@@ -58,12 +61,14 @@
2976
58
  exit 1
61
  exit 1
2977
59
fi
62
fi
2978
60
63
2979
64
echo "$(date -R): Initiating archive publishing operations..."
2980
65
2981
61
DONEPUB=no
66
DONEPUB=no
2982
62
67
2983
63
cleanup () {
68
cleanup () {
2985
64
  echo "Cleaning up lockfile."
69
  echo "$(date -R): Cleaning up lockfile."
2986
65
  rm -f $LOCKFILE
70
  rm -f $LOCKFILE
2988
66
  echo "Moving dists backup to safe keeping for next time."
71
  echo "$(date -R): Moving dists backup to safe keeping for next time."
2989
67
  if [ "x$DONEPUB" = "xyes" ]; then
72
  if [ "x$DONEPUB" = "xyes" ]; then
2990
68
    if [ -d ${DISTSROOT}.old ]; then
73
    if [ -d ${DISTSROOT}.old ]; then
2991
69
      mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
74
      mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
2992
@@ -89,11 +94,13 @@
2993
89
# Lock claimed.
94
# Lock claimed.
2994
90
95
2995
91
# Process the accepted queue into the publishing records.
96
# Process the accepted queue into the publishing records.
2996
97
echo "$(date -R): Processing the accepted queue into the publishing records..."
2997
92
process-accepted.py -v -v -v $DISTRONAME
98
process-accepted.py -v -v -v $DISTRONAME
2998
93
99
2999
94
# If doing a security run, find out which suites are pending publication.
100
# If doing a security run, find out which suites are pending publication.
3000
95
SUITEOPTS=""
101
SUITEOPTS=""
3001
96
if [ "$SECURITY_PUBLISHER" = "yes" ]; then
102
if [ "$SECURITY_PUBLISHER" = "yes" ]; then
3002
103
    echo "$(date -R): Querying which suites are pending publication..."
3003
97
    SUITES=$(lp-query-distro.py pending_suites)
104
    SUITES=$(lp-query-distro.py pending_suites)
3004
98
    SECURITY_SUITES="no"
105
    SECURITY_SUITES="no"
3005
99
    if [ -n "$SUITES" ]; then
106
    if [ -n "$SUITES" ]; then
3006
@@ -109,7 +116,7 @@
3007
109
        done
116
        done
3008
110
    fi
117
    fi
3009
111
    if [ "$SECURITY_SUITES" != "yes" ]; then
118
    if [ "$SECURITY_SUITES" != "yes" ]; then
3011
112
        echo "Nothing to do for security publisher; exiting."
119
        echo "$(date -R): Nothing to do for security publisher; exiting."
3012
113
        exit 0
120
        exit 0
3013
114
    fi
121
    fi
3014
115
fi
122
fi
3015
@@ -122,21 +129,27 @@
3016
122
# This should achieve the same as copying, only faster.
129
# This should achieve the same as copying, only faster.
3017
123
130
3018
124
# Create backup dists folder, if this is the first time.
131
# Create backup dists folder, if this is the first time.
3019
132
echo "$(date -R): Creating backup dists directories..."
3020
125
mkdir -p ${DISTSCOPYROOT}/dists
133
mkdir -p ${DISTSCOPYROOT}/dists
3021
126
mkdir -p ${DISTSCOPYROOT_PARTNER}/dists
134
mkdir -p ${DISTSCOPYROOT_PARTNER}/dists
3022
127
135
3023
128
# Move the backup dists folder into place.
136
# Move the backup dists folder into place.
3024
137
echo "$(date -R): Moving backup dists into place..."
3025
129
mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
138
mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
3026
130
mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new
139
mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new
3027
131
140
3028
132
# Bring it up-to-date efficiently with rsync. --delete is required to
141
# Bring it up-to-date efficiently with rsync. --delete is required to
3029
133
# ensure we don't ressurect things previously deleted, bug 58835.
142
# ensure we don't ressurect things previously deleted, bug 58835.
3030
143
echo "$(date -R): Updating dists directories..."
3031
134
rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
144
rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
3032
135
rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new
145
rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new
3033
136
146
3034
137
# Publish the results for all archives (except PPA).
147
# Publish the results for all archives (except PPA).
3035
138
# The -R only affects the primary and the partner archive.
148
# The -R only affects the primary and the partner archive.
3036
149
echo "$(date -R): Publishing the $DISTRONAME partner archive..."
3037
139
publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new
150
publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new
3038
151
3039
152
echo "$(date -R): Publishing the $DISTRONAME archive..."
3040
140
publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new
153
publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new
3041
141
154
3042
142
set +x
155
set +x
3043
@@ -148,39 +161,40 @@
3044
148
       $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
161
       $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
3045
149
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
162
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
3046
150
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
163
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
3048
151
    echo "(re-)signing $CANDIDATE"
164
    echo "$(date -R): (re-)signing $CANDIDATE"
3049
152
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
165
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
3050
153
  else
166
  else
3052
154
    echo "Not re-signing $CANDIDATE"
167
    echo "$(date -R): Not re-signing $CANDIDATE"
3053
155
  fi
168
  fi
3054
156
done
169
done
3055
157
SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
170
SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
3056
158
for CANDIDATE in $SIGNLIST_PARTNER; do
171
for CANDIDATE in $SIGNLIST_PARTNER; do
3057
159
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
172
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
3058
160
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
173
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
3060
161
    echo "(re-)signing $CANDIDATE"
174
    echo "$(date -R): (re-)signing $CANDIDATE"
3061
162
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
175
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
3062
163
  else
176
  else
3064
164
    echo "Not re-signing $CANDIDATE"
177
    echo "$(date -R): Not re-signing $CANDIDATE"
3065
165
  fi
178
  fi
3066
166
done
179
done
3067
167
180
3068
168
# The Packages and Sources files are very large and would cripple our
181
# The Packages and Sources files are very large and would cripple our
3069
169
# mirrors, so we remove them now that the uncompressed MD5SUMS are in the
182
# mirrors, so we remove them now that the uncompressed MD5SUMS are in the
3070
170
# Release files.
183
# Release files.
3072
171
echo Removing uncompressed Packages and Sources files
184
echo "$(date -R): Removing uncompressed Packages and Sources files"
3073
172
find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
185
find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
3074
173
find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
186
find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
3075
174
187
3076
175
# Copy in the indices.
188
# Copy in the indices.
3077
176
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
189
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
3079
177
    echo Copying the indices into place.
190
    echo "$(date -R): Copying the indices into place."
3080
178
    rm -f $INDICES/override.*
191
    rm -f $INDICES/override.*
3081
179
    cp $OVERRIDEROOT/override.* $INDICES
192
    cp $OVERRIDEROOT/override.* $INDICES
3082
180
fi
193
fi
3083
181
194
3084
182
# As close to atomically as possible, put the new dists into place for the
195
# As close to atomically as possible, put the new dists into place for the
3085
183
# primary and partner archives.
196
# primary and partner archives.
3086
197
echo "$(date -R): Placing the new dists into place..."
3087
184
mv $DISTSROOT ${DISTSROOT}.old
198
mv $DISTSROOT ${DISTSROOT}.old
3088
185
mv ${DISTSROOT}.new $DISTSROOT
199
mv ${DISTSROOT}.new $DISTSROOT
3089
186
mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
200
mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
3090
@@ -192,19 +206,21 @@
3091
192
# Generate the -commercial pocket for backwards compatibility with
206
# Generate the -commercial pocket for backwards compatibility with
3092
193
# dapper, edgy and feisty releases.  Don't fail the whole script if it
207
# dapper, edgy and feisty releases.  Don't fail the whole script if it
3093
194
# fails.
208
# fails.
3094
209
echo "$(date -R): Generating -commerical pocket..."
3095
195
commercial-compat.sh || true
210
commercial-compat.sh || true
3096
196
211
3098
197
# Touch everything you asked us to do.
212
# Timestamp our trace file to track when the last archive publisher run took
3099
213
# place.
3100
198
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
214
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
3104
199
    for FILE in $TOUCHLIST; do
215
    echo "$(date -R): Timestamping trace file..."
3105
200
      touch "$FILE"
216
    date -u > "$TRACEFILE"
3103
201
    done
3106
202
fi
217
fi
3107
203
218
3108
204
# Skip long-running processes when doing a quick security run.
219
# Skip long-running processes when doing a quick security run.
3109
205
if [ "$SECURITY_PUBLISHER" != "yes" ]; then
220
if [ "$SECURITY_PUBLISHER" != "yes" ]; then
3110
206
221
3111
207
    # Make the lslr because we all love those.
222
    # Make the lslr because we all love those.
3112
223
    echo "$(date -R): Creating ls-lR.gz..."
3113
208
    LSLR=ls-lR.gz
224
    LSLR=ls-lR.gz
3114
209
    ( cd $ARCHIVEROOT ; \
225
    ( cd $ARCHIVEROOT ; \
3115
210
      rm -f .$LSLR.new ; \
226
      rm -f .$LSLR.new ; \
3116
@@ -216,25 +232,35 @@
3117
216
      mv -f .$LSLR.new $LSLR )
232
      mv -f .$LSLR.new $LSLR )
3118
217
233
3119
218
    # Run dsync over primary archive only.
234
    # Run dsync over primary archive only.
3120
235
    echo "$(date -R): Running dsync over primary archive..."
3121
219
    ( cd $ARCHIVEROOT ; \
236
    ( cd $ARCHIVEROOT ; \
3122
220
      dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
237
      dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
3123
221
      (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
238
      (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
3124
222
      dsync-flist -q link-dups $DSYNCLIST || true )
239
      dsync-flist -q link-dups $DSYNCLIST || true )
3125
223
240
3126
224
    # Clear out empty and thus redundant dirs.
241
    # Clear out empty and thus redundant dirs.
3127
242
    echo "$(date -R): Clearing out empty directories..."
3128
225
    find $ARCHIVEROOT -type d -empty | xargs -r rmdir
243
    find $ARCHIVEROOT -type d -empty | xargs -r rmdir
3129
226
    find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir
244
    find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir
3130
227
245
3131
228
    # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
246
    # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
3132
229
    # the mirrors should be always triggered.
247
    # the mirrors should be always triggered.
3134
230
    cron.germinate || echo "cron.germinate failed with exit code $?" >&2
248
    echo "$(date -R): Running cron.germinate..."
3135
249
    cron.germinate || echo "$(date -R): cron.germinate failed with exit code $?"
3136
231
250
3137
232
# End of block skipped by security publishing.
251
# End of block skipped by security publishing.
3138
233
fi
252
fi
3139
234
253
3140
235
# Trigger master mirrors.
254
# Trigger master mirrors.
3141
236
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
255
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
3145
237
    ssh archvsync@syowa
256
    echo "$(date -R): Triggering master mirrors..."
3146
238
    ssh archvsync@frei
257
3147
239
    ssh archvsync@rockhopper
258
    for HOST in $MASTERMIRRORS; do
3148
259
        echo "$(date -R): Triggering $HOST:"
3149
260
        ssh archvsync@$HOST
3150
261
    done
3151
262
3152
263
    echo "$(date -R): Master mirror triggers completed."
3153
240
fi
264
fi
3154
265
3155
266
echo "$(date -R): Archive publishing operations completed."
3156
241
267
3157
=== added file 'cronscripts/publishing/maintenance-check.py'
3158
--- cronscripts/publishing/maintenance-check.py	1970-01-01 00:00:00 +0000
3159
+++ cronscripts/publishing/maintenance-check.py	2010-05-05 01:58:48 +0000
3160
@@ -0,0 +1,382 @@
3161
1
#!/usr/bin/python
3162
2
#
3163
3
# python port of the nice maintainace-check script by  Nick Barcet
3164
4
# 
3165
5
# taken from:
3166
6
#  https://code.edge.launchpad.net/~mvo/ubuntu-maintenance-check/python-port
3167
7
# (where it will vanish once taken here)
3168
8
3169
9
# this warning filter is only needed on older versions of python-apt,
3170
10
# once the machine runs lucid it can be removed
3171
11
import warnings
3172
12
warnings.filterwarnings("ignore","apt API not stable yet")
3173
13
import apt
3174
14
warnings.resetwarnings()
3175
15
3176
16
import apt_pkg
3177
17
import logging
3178
18
import os
3179
19
import sys
3180
20
import urllib2
3181
21
import urlparse
3182
22
3183
23
from optparse import OptionParser
3184
24
3185
25
# This is fun! We have a bunch of cases for 10.04 LTS
3186
26
#
3187
27
#  - distro "ubuntu" follows SUPPORT_TIMEFRAME_LTS but only for
3188
28
#    amd64/i386
3189
29
#  - distros "kubuntu", "edubuntu" and "netbook" need to be
3190
30
#    considered *but* only follow SUPPORT_TIMEFRAME
3191
31
#  - anything that is in armel follows SUPPORT_TIMEFRAME
3192
32
#  
3193
33
3194
34
# codename of the lts releases
3195
35
LTS_RELEASES = [ "dapper", "hardy", "lucid" ]
3196
36
3197
37
# architectures that are full supported (including LTS time)
3198
38
PRIMARY_ARCHES =  ["i386", "amd64"]
3199
39
3200
40
# architectures we support (but not for LTS time)
3201
41
SUPPORTED_ARCHES = PRIMARY_ARCHES + ["armel"]
3202
42
3203
43
# what defines the seeds is documented in wiki.ubuntu.com/SeedManagement
3204
44
SERVER_SEEDS = [ "supported-server", "server-ship"]
3205
45
DESKTOP_SEEDS = ["ship", "supported-desktop", "supported-desktop-extra"]
3206
46
SUPPORTED_SEEDS = [ "all" ]
3207
47
3208
48
# normal support timeframe
3209
49
# time, seeds, arches
3210
50
SUPPORT_TIMEFRAME = [
3211
51
    ("18m", SUPPORTED_SEEDS),
3212
52
]
3213
53
3214
54
# lts support timeframe
3215
55
# time, seeds, arches
3216
56
SUPPORT_TIMEFRAME_LTS = [
3217
57
    ("5y", SERVER_SEEDS),
3218
58
    ("3y", DESKTOP_SEEDS),
3219
59
    ("18m", SUPPORTED_SEEDS),
3220
60
]
3221
61
3222
62
# distro names and if they get LTS support (order is important)
3223
63
DISTRO_NAMES_AND_LTS_SUPPORT = [ ("ubuntu",   True),
3224
64
                                 ("kubuntu",  True),
3225
65
                                 ("netbook",  False),
3226
66
                               ]
3227
67
3228
68
# germinate output base directory
3229
69
BASE_URL = "http://people.canonical.com/~ubuntu-archive/germinate-output/"
3230
70
3231
71
# hints dir url, hints file is "$distro.hints" by default
3232
72
# (e.g. lucid.hints)
3233
73
HINTS_DIR_URL = "http://people.canonical.com/~ubuntu-archive/seeds/platform.%s/SUPPORTED_HINTS"
3234
74
3235
75
# we need the archive root to parse the Sources file to support
3236
76
# by-source hints
3237
77
ARCHIVE_ROOT = "http://archive.ubuntu.com/ubuntu"
3238
78
3239
79
# support timeframe tag used in the Packages file
3240
80
SUPPORT_TAG = "Supported"
3241
81
3242
82
def get_binaries_for_source_pkg(srcname):
3243
83
    """ Return all binary package names for the given source package name.
3244
84
3245
85
    :param srcname: The source package name.
3246
86
    :return: A list of binary package names.
3247
87
    """
3248
88
    pkgnames = set()
3249
89
    recs = apt_pkg.GetPkgSrcRecords()
3250
90
    while recs.Lookup(srcname):
3251
91
        for binary in recs.Binaries:
3252
92
            pkgnames.add(binary)
3253
93
    return pkgnames
3254
94
3255
95
def expand_src_pkgname(pkgname):
3256
96
    """ Expand a package name if it is prefixed with src.
3257
97
3258
98
    If the package name is prefixed with src it will be expanded
3259
99
    to a list of binary package names. Otherwise the original
3260
100
    package name will be returned.
3261
101
    
3262
102
    :param pkgname: The package name (that may include src:prefix).
3263
103
    :return: A list of binary package names (the list may be one element long).
3264
104
    """
3265
105
    if not pkgname.startswith("src:"):
3266
106
        return [pkgname]
3267
107
    return get_binaries_for_source_pkg(pkgname.split("src:")[1])
3268
108
3269
109
def create_and_update_deb_src_source_list(distroseries):
3270
110
    """ Create sources.list and update cache.
3271
111
3272
112
    This creates a sources.list file with deb-src entries for a given 
3273
113
    distroseries and apt.Cache.update() to make sure the data is up-to-date.
3274
114
    :param distro: The code name of the distribution series (e.g. lucid).
3275
115
    :return: None
3276
116
    :raises: IOError: When cache update fails.
3277
117
    """
3278
118
    # apt root dir
3279
119
    rootdir="./aptroot.%s" % distroseries
3280
120
    sources_list_dir = os.path.join(rootdir, "etc","apt")
3281
121
    if not os.path.exists(sources_list_dir):
3282
122
        os.makedirs(sources_list_dir)
3283
123
    sources_list = open(os.path.join(sources_list_dir, "sources.list"),"w")
3284
124
    for pocket in [
3285
125
        "%s" % distroseries, 
3286
126
        "%s-updates" % distroseries, 
3287
127
        "%s-security" % distroseries]:
3288
128
        sources_list.write(
3289
129
            "deb-src %s %s main restricted\n" % (
3290
130
                ARCHIVE_ROOT, pocket))
3291
131
        sources_list.write(
3292
132
            "deb %s %s main restricted\n" % (
3293
133
                ARCHIVE_ROOT, pocket))
3294
134
    sources_list.close()
3295
135
    # create required dirs/files for apt.Cache(rootdir) to work on older
3296
136
    # versions of python-apt. once lucid is used it can be removed
3297
137
    for d in  ["var/lib/dpkg", 
3298
138
               "var/cache/apt/archives/partial",
3299
139
               "var/lib/apt/lists/partial"]:
3300
140
        if not os.path.exists(os.path.join(rootdir,d)):
3301
141
            os.makedirs(os.path.join(rootdir,d))
3302
142
    if not os.path.exists(os.path.join(rootdir,"var/lib/dpkg/status")):
3303
143
        open(os.path.join(rootdir,"var/lib/dpkg/status"),"w")
3304
144
    # open cache with our just prepared rootdir
3305
145
    cache = apt.Cache(rootdir=rootdir)
3306
146
    try:
3307
147
        cache.update(apt.progress.FetchProgress())
3308
148
    except SystemError:
3309
149
        logging.exception("cache.update() failed")
3310
150
3311
151
def get_structure(distroname, version):
3312
152
    """ Get structure file conent for named distro and distro version.
3313
153
    
3314
154
    :param name: Name of the distribution (e.g. kubuntu, ubuntu, xubuntu).
3315
155
    :param version: Code name of the distribution version (e.g. lucid).
3316
156
    :return: List of strings with the structure file content
3317
157
    """
3318
158
    f = urllib2.urlopen("%s/%s.%s/structure" % (BASE_URL, distroname, version))
3319
159
    structure = f.readlines()
3320
160
    f.close()
3321
161
    return structure
3322
162
3323
163
def expand_seeds(structure, seedname):
3324
164
    """ Expand seed by its dependencies using the strucure file.
3325
165
3326
166
    :param structure: The content of the STRUCTURE file as string list.
3327
167
    :param seedname: The name of the seed as string that needs to be expanded.
3328
168
    :return: a set() for the seed dependencies (excluding the original seedname)
3329
169
    """
3330
170
    seeds = []
3331
171
    for line in structure:
3332
172
        if line.startswith("%s:" % seedname):
3333
173
            seeds += line.split(":")[1].split()
3334
174
            for seed in seeds:
3335
175
                seeds += expand_seeds(structure, seed)
3336
176
    return set(seeds)
3337
177
3338
178
def get_packages_for_seeds(name, distro, seeds):
3339
179
    """
3340
180
    get packages for the given name (e.g. ubuntu) and distro release 
3341
181
    (e.g. lucid) that are in the given list of seeds
3342
182
    returns a set() of package names
3343
183
    """
3344
184
    pkgs_in_seeds = {}
3345
185
    for bseed in seeds:
3346
186
        for seed in [bseed]: #, bseed+".build-depends", bseed+".seed"]:
3347
187
            pkgs_in_seeds[seed] = set()
3348
188
            seedurl = "%s/%s.%s/%s" % (BASE_URL,name, distro, seed)
3349
189
            logging.debug("looking for '%s'" % seedurl)
3350
190
            try:
3351
191
                f = urllib2.urlopen(seedurl)
3352
192
                for line in f:
3353
193
                    # ignore lines that are not a package name (headers etc)
3354
194
                    if line[0] < 'a' or line[0] > 'z':
3355
195
                        continue
3356
196
                    # lines are (package,source,why,maintainer,size,inst-size)
3357
197
                    if options.source_packages:
3358
198
                        pkgname = line.split("|")[1]
3359
199
                    else:
3360
200
                        pkgname = line.split("|")[0]
3361
201
                    pkgs_in_seeds[seed].add(pkgname.strip())
3362
202
                f.close()
3363
203
            except Exception, e:
3364
204
                logging.error("seed %s failed (%s)" % (seedurl, e))
3365
205
    return pkgs_in_seeds
3366
206
3367
207
def what_seeds(pkgname, seeds):
3368
208
    in_seeds = set()
3369
209
    for s in seeds:
3370
210
        if pkgname in seeds[s]:
3371
211
            in_seeds.add(s)
3372
212
    return in_seeds
3373
213
3374
214
def compare_support_level(x, y):
3375
215
    """
3376
216
    compare two support level strings of the form 18m, 3y etc
3377
217
    :parm x: the first support level
3378
218
    :parm y: the second support level
3379
219
    :return: negative if x < y, zero if x==y, positive if x > y
3380
220
    """
3381
221
    def support_to_int(support_time):
3382
222
        """
3383
223
        helper that takes a support time string and converts it to 
3384
224
        a integer for cmp()
3385
225
        """
3386
226
        # allow strings like "5y (kubuntu-common)
3387
227
        x = support_time.split()[0]
3388
228
        if x.endswith("y"):
3389
229
            return 12 * int(x[0:-1])
3390
230
        elif x.endswith("m"):
3391
231
            return int(x[0:-1])
3392
232
        else:
3393
233
            raise ValueError("support time '%s' has to end with y or m" % x)
3394
234
    return cmp(support_to_int(x), support_to_int(y))
3395
235
3396
236
def get_packages_support_time(structure, name, pkg_support_time, support_timeframe_list):
3397
237
    """
3398
238
    input a structure file and a list of pair<timeframe, seedlist>
3399
239
    return a dict of pkgnames -> support timeframe string
3400
240
    """
3401
241
    for (timeframe, seedlist) in support_timeframe_list:
3402
242
        expanded = set()
3403
243
        for s in seedlist:
3404
244
            expanded.add(s)
3405
245
            expanded |= expand_seeds(structure, s)
3406
246
        pkgs_in_seeds = get_packages_for_seeds(name, distro, expanded)
3407
247
        for seed in pkgs_in_seeds:
3408
248
            for pkg in pkgs_in_seeds[seed]:
3409
249
                if not pkg in pkg_support_time:
3410
250
                    pkg_support_time[pkg] = timeframe
3411
251
                else:
3412
252
                    old_timeframe = pkg_support_time[pkg]
3413
253
                    if compare_support_level(old_timeframe, timeframe) < 0:
3414
254
                        logging.debug("overwriting %s from %s to %s" % (
3415
255
                                pkg, old_timeframe, timeframe))
3416
256
                        pkg_support_time[pkg] = timeframe
3417
257
                if options.with_seeds:
3418
258
                    pkg_support_time[pkg] += " (%s)" % ", ".join(what_seeds(pkg, pkgs_in_seeds))
3419
259
3420
260
3421
261
    return pkg_support_time
3422
262
3423
263
if __name__ == "__main__":
3424
264
    parser = OptionParser()
3425
265
    parser.add_option("--with-seeds", "", default=False,
3426
266
                      action="store_true", 
3427
267
                      help="add seed(s) of the package that are responsible for the maintaince time")
3428
268
    parser.add_option("--source-packages", "", default=False,
3429
269
                      action="store_true", 
3430
270
                      help="show as source pkgs")
3431
271
    parser.add_option("--hints-file", "", default=None,
3432
272
                      help="use diffenrt use hints file location")
3433
273
    (options, args) = parser.parse_args()
3434
274
3435
275
    # init
3436
276
    if len(args) > 0:
3437
277
        distro = args[0]
3438
278
        if distro[0] < 'h':
3439
279
            print "ERROR: only hardy or later is supported"
3440
280
            sys.exit(1)
3441
281
    else:
3442
282
        distro = "lucid"
3443
283
3444
284
    # make sure our deb-src information is up-to-date
3445
285
    create_and_update_deb_src_source_list(distro)
3446
286
3447
287
    if options.hints_file:
3448
288
        hints_file = options.hints_file
3449
289
        (schema, netloc, path, query, fragment) = urlparse.urlsplit(hints_file)
3450
290
        if not schema:
3451
291
            hints_file = "file:%s" % path
3452
292
    else:
3453
293
        hints_file = HINTS_DIR_URL % distro
3454
294
        
3455
295
    # go over the distros we need to check
3456
296
    pkg_support_time = {}
3457
297
    for (name, lts_supported) in DISTRO_NAMES_AND_LTS_SUPPORT:
3458
298
3459
299
        # get basic structure file
3460
300
        structure = get_structure(name, distro)
3461
301
    
3462
302
        # get dicts of pkgname -> support timeframe string
3463
303
        support_timeframe = SUPPORT_TIMEFRAME
3464
304
        if lts_supported and distro in LTS_RELEASES:
3465
305
            support_timeframe =  SUPPORT_TIMEFRAME_LTS
3466
306
        else:
3467
307
            support_timeframe = SUPPORT_TIMEFRAME
3468
308
        get_packages_support_time(structure, name, pkg_support_time, support_timeframe)
3469
309
3470
310
    # now go over the bits in main that we have not seen (because
3471
311
    # they are not in any seed and got added manually into "main"
3472
312
    for arch in PRIMARY_ARCHES:
3473
313
        rootdir="./aptroot.%s" % distro
3474
314
        apt_pkg.Config.Set("APT::Architecture", arch)
3475
315
        cache = apt.Cache(rootdir=rootdir)
3476
316
        try:
3477
317
            cache.update(apt.progress.FetchProgress())
3478
318
        except SystemError:
3479
319
            logging.exception("cache.update() failed")
3480
320
        cache.open(apt.progress.OpProgress())
3481
321
        for pkg in cache:
3482
322
            if not pkg.name in pkg_support_time:
3483
323
                pkg_support_time[pkg.name] = support_timeframe[-1][0]
3484
324
                logging.warn("add package in main but not in seeds %s with %s" % 
3485
325
                             (pkg.name, pkg_support_time[pkg.name]))
3486
326
3487
327
    # now check the hints file that is used to overwrite 
3488
328
    # the default seeds
3489
329
    try:
3490
330
        for line in urllib2.urlopen(hints_file):
3491
331
            line = line.strip()
3492
332
            if not line or line.startswith("#"):
3493
333
                continue
3494
334
            try:
3495
335
                (raw_pkgname, support_time) = line.split()
3496
336
                for pkgname in expand_src_pkgname(raw_pkgname):
3497
337
                    if support_time == 'unsupported':
3498
338
                        try:
3499
339
                            del pkg_support_time[pkgname]
3500
340
                            sys.stderr.write("hints-file: marking %s unsupported\n" % pkgname)
3501
341
                        except KeyError:
3502
342
                            pass
3503
343
                    else:
3504
344
                        if pkg_support_time.get(pkgname) != support_time:
3505
345
                            sys.stderr.write(
3506
346
                                "hints-file: changing %s from %s to %s\n" % (
3507
347
                                    pkgname,  pkg_support_time.get(pkgname), 
3508
348
                                    support_time))
3509
349
                            pkg_support_time[pkgname] = support_time
3510
350
            except:
3511
351
                logging.exception("can not parse line '%s'" % line)
3512
352
    except urllib2.HTTPError, e:
3513
353
        if e.getcode() != 404:
3514
354
            raise
3515
355
        sys.stderr.write("hints-file: %s gave 404 error\n" % hints_file)
3516
356
    
3517
357
    # output suitable for the extra-override file
3518
358
    for pkgname in sorted(pkg_support_time.keys()):
3519
359
        # special case, the hints file may contain overrides that
3520
360
        # are arch-specific (like zsh-doc/armel)
3521
361
        if "/" in pkgname:
3522
362
            print "%s %s %s" % (
3523
363
                pkgname, SUPPORT_TAG, pkg_support_time[pkgname])
3524
364
        else:
3525
365
            # go over the supported arches, they are divided in 
3526
366
            # first-class (PRIMARY) and second-class with different
3527
367
            # support levels
3528
368
            for arch in SUPPORTED_ARCHES:
3529
369
                # ensure we do not overwrite arch-specific overwrites
3530
370
                pkgname_and_arch = "%s/%s" % (pkgname, arch)
3531
371
                if pkgname_and_arch in pkg_support_time:
3532
372
                    break
3533
373
                if arch in PRIMARY_ARCHES:
3534
374
                    # arch with full LTS support
3535
375
                    print "%s %s %s" % (
3536
376
                        pkgname_and_arch, SUPPORT_TAG, pkg_support_time[pkgname])
3537
377
                else:
3538
378
                    # not a LTS supported architecture, gets only regular
3539
379
                    # support_timeframe
3540
380
                    print "%s %s %s" % (
3541
381
                        pkgname_and_arch, SUPPORT_TAG, SUPPORT_TIMEFRAME[0][0])
3542
382
                
3543
0
383
3544
=== modified file 'cronscripts/reclaimbranchspace.py'
3545
--- cronscripts/reclaimbranchspace.py	2009-09-03 20:06:45 +0000
3546
+++ cronscripts/reclaimbranchspace.py	2010-05-05 01:58:48 +0000
3547
@@ -1,4 +1,4 @@
3549
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3550
2
#
2
#
3551
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3552
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3553
5
5
3554
=== modified file 'cronscripts/rosetta-approve-imports.py'
3555
--- cronscripts/rosetta-approve-imports.py	2009-09-26 06:21:08 +0000
3556
+++ cronscripts/rosetta-approve-imports.py	2010-05-05 01:58:48 +0000
3557
@@ -1,4 +1,4 @@
3559
1
#! /usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3560
2
#
2
#
3561
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3562
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3563
5
5
3564
=== modified file 'cronscripts/rosetta-branches.py'
3565
--- cronscripts/rosetta-branches.py	2009-09-03 20:29:25 +0000
3566
+++ cronscripts/rosetta-branches.py	2010-05-05 01:58:48 +0000
3567
@@ -1,4 +1,4 @@
3569
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3570
2
#
2
#
3571
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3572
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3573
@@ -17,7 +17,7 @@
3574
17
from zope.component import getUtility
17
from zope.component import getUtility
3575
18
18
3576
19
from canonical.config import config
19
from canonical.config import config
3578
20
from lp.codehosting.vfs.branchfs import get_scanner_server
20
from lp.codehosting.vfs.branchfs import get_ro_server
3579
21
from lp.services.job.runner import JobRunner
21
from lp.services.job.runner import JobRunner
3580
22
from lp.code.interfaces.branchjob import IRosettaUploadJobSource
22
from lp.code.interfaces.branchjob import IRosettaUploadJobSource
3581
23
from lp.services.scripts.base import LaunchpadCronScript
23
from lp.services.scripts.base import LaunchpadCronScript
3582
@@ -31,12 +31,12 @@
3583
31
        globalErrorUtility.configure('rosettabranches')
31
        globalErrorUtility.configure('rosettabranches')
3584
32
        runner = JobRunner.fromReady(
32
        runner = JobRunner.fromReady(
3585
33
            getUtility(IRosettaUploadJobSource), self.logger)
33
            getUtility(IRosettaUploadJobSource), self.logger)
3588
34
        server = get_scanner_server()
34
        server = get_ro_server()
3589
35
        server.setUp()
35
        server.start_server()
3590
36
        try:
36
        try:
3591
37
            runner.runAll()
37
            runner.runAll()
3592
38
        finally:
38
        finally:
3594
39
            server.tearDown()
39
            server.stop_server()
3595
40
        self.logger.info('Ran %d RosettaBranchJobs.',
40
        self.logger.info('Ran %d RosettaBranchJobs.',
3596
41
                         len(runner.completed_jobs))
41
                         len(runner.completed_jobs))
3597
42
42
3598
43
43
3599
=== modified file 'cronscripts/rosetta-export-queue.py'
3600
--- cronscripts/rosetta-export-queue.py	2009-07-17 00:26:05 +0000
3601
+++ cronscripts/rosetta-export-queue.py	2010-05-05 01:58:48 +0000
3602
@@ -1,4 +1,4 @@
3604
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3605
2
#
2
#
3606
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3607
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3608
5
5
3609
=== modified file 'cronscripts/rosetta-pofile-stats-daily.py'
3610
--- cronscripts/rosetta-pofile-stats-daily.py	2009-08-07 17:58:19 +0000
3611
+++ cronscripts/rosetta-pofile-stats-daily.py	2010-05-05 01:58:48 +0000
3612
@@ -1,4 +1,4 @@
3614
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3615
2
#
2
#
3616
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3617
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3618
5
5
3619
=== modified file 'cronscripts/rosetta-pofile-stats.py'
3620
--- cronscripts/rosetta-pofile-stats.py	2009-08-06 12:30:58 +0000
3621
+++ cronscripts/rosetta-pofile-stats.py	2010-05-05 01:58:48 +0000
3622
@@ -1,4 +1,4 @@
3624
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3625
2
#
2
#
3626
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3627
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3628
5
5
3629
=== modified file 'cronscripts/rosetta-poimport.py'
3630
--- cronscripts/rosetta-poimport.py	2009-09-29 15:33:30 +0000
3631
+++ cronscripts/rosetta-poimport.py	2010-05-05 01:58:48 +0000
3632
@@ -1,4 +1,4 @@
3634
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3635
2
#
2
#
3636
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3637
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3638
5
5
3639
=== added file 'cronscripts/scan_branches.py'
3640
--- cronscripts/scan_branches.py	1970-01-01 00:00:00 +0000
3641
+++ cronscripts/scan_branches.py	2010-05-05 01:58:48 +0000
3642
@@ -0,0 +1,25 @@
3643
1
#!/usr/bin/python2.5 -S
3644
2
#
3645
3
# Copyright 2010 Canonical Ltd.  This software is licensed under the
3646
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3647
5
3648
6
"""Scan branches for new revisions."""
3649
7
3650
8
__metaclass__ = type
3651
9
3652
10
import _pythonpath
3653
11
3654
12
from lp.services.job.runner import JobCronScript
3655
13
from lp.code.interfaces.branchjob import IBranchScanJobSource
3656
14
3657
15
3658
16
class RunScanBranches(JobCronScript):
3659
17
    """Run BranchScanJob jobs."""
3660
18
3661
19
    config_name = 'branchscanner'
3662
20
    source_interface = IBranchScanJobSource
3663
21
3664
22
3665
23
if __name__ == '__main__':
3666
24
    script = RunScanBranches()
3667
25
    script.lock_and_run()
3668
0
26
3669
=== modified file 'cronscripts/send-bug-notifications.py'
3670
--- cronscripts/send-bug-notifications.py	2009-06-24 20:52:01 +0000
3671
+++ cronscripts/send-bug-notifications.py	2010-05-05 01:58:48 +0000
3672
@@ -1,4 +1,4 @@
3674
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3675
2
#
2
#
3676
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3677
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3678
5
5
3679
=== modified file 'cronscripts/send-person-notifications.py'
3680
--- cronscripts/send-person-notifications.py	2009-06-24 20:52:01 +0000
3681
+++ cronscripts/send-person-notifications.py	2010-05-05 01:58:48 +0000
3682
@@ -1,4 +1,4 @@
3684
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3685
2
#
2
#
3686
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3687
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3688
@@ -14,15 +14,11 @@
3689
14
__metaclass__ = type
14
__metaclass__ = type
3690
15
15
3691
16
import _pythonpath
16
import _pythonpath
3692
17
from datetime import timedelta, datetime
3693
18
import pytz
3694
19
3695
20
from zope.component import getUtility
3696
21
17
3697
22
from canonical.config import config
18
from canonical.config import config
3700
23
from canonical.launchpad.interfaces.personnotification import (
19
3699
24
    IPersonNotificationSet)
3701
25
from lp.services.scripts.base import LaunchpadCronScript
20
from lp.services.scripts.base import LaunchpadCronScript
3702
21
from lp.registry.scripts.personnotification import PersonNotificationManager
3703
26
22
3704
27
23
3705
28
class SendPersonNotifications(LaunchpadCronScript):
24
class SendPersonNotifications(LaunchpadCronScript):
3706
@@ -36,39 +32,9 @@
3707
36
    """
32
    """
3708
37
33
3709
38
    def main(self):
34
    def main(self):
3743
39
        notifications_sent = False
35
        manager = PersonNotificationManager(self.txn, self.logger)
3744
40
        notification_set = getUtility(IPersonNotificationSet)
36
        unsent_notifications = manager.sendNotifications()
3745
41
        pending_notifications = notification_set.getNotificationsToSend()
37
        manager.purgeNotifications(unsent_notifications)
3713
42
        self.logger.info(
3714
43
            '%d notification(s) to send.' % pending_notifications.count())
3715
44
        for notification in pending_notifications:
3716
45
            person = notification.person
3717
46
            self.logger.info(
3718
47
                "Sending notification to %s <%s>."
3719
48
                % (person.name, person.preferredemail.email))
3720
49
            notification.send()
3721
50
            notifications_sent = True
3722
51
            # Commit after each email sent, so that we won't re-mail the
3723
52
            # notifications in case of something going wrong in the middle.
3724
53
            self.txn.commit()
3725
54
3726
55
        if not notifications_sent:
3727
56
            self.logger.debug("No notifications were sent.")
3728
57
3729
58
        # Delete PersonNotifications that are older than the retention
3730
59
        # limit set in the configuration.
3731
60
        retained_days = timedelta(
3732
61
            days=int(config.person_notification.retained_days))
3733
62
        time_limit = (datetime.now(pytz.timezone('UTC')) - retained_days)
3734
63
        to_delete = notification_set.getNotificationsOlderThan(time_limit)
3735
64
        if to_delete.count():
3736
65
            self.logger.info(
3737
66
                "Notification retention limit is %s." % retained_days)
3738
67
            self.logger.info(
3739
68
                "Deleting %d old notification(s)." % to_delete.count())
3740
69
            for notification in to_delete:
3741
70
                notification.destroySelf()
3742
71
            self.txn.commit()
3746
72
38
3747
73
39
3748
74
if __name__ == '__main__':
40
if __name__ == '__main__':
3749
75
41
3750
=== modified file 'cronscripts/sendbranchmail.py'
3751
--- cronscripts/sendbranchmail.py	2009-09-03 19:25:57 +0000
3752
+++ cronscripts/sendbranchmail.py	2010-05-05 01:58:48 +0000
3753
@@ -1,4 +1,4 @@
3755
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3756
2
#
2
#
3757
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3758
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3759
@@ -16,7 +16,7 @@
3760
16
from zope.component import getUtility
16
from zope.component import getUtility
3761
17
17
3762
18
from canonical.config import config
18
from canonical.config import config
3764
19
from lp.codehosting.vfs import get_scanner_server
19
from lp.codehosting.vfs import get_ro_server
3765
20
from lp.services.job.runner import JobRunner
20
from lp.services.job.runner import JobRunner
3766
21
from lp.code.interfaces.branchjob import (
21
from lp.code.interfaces.branchjob import (
3767
22
    IRevisionMailJobSource, IRevisionsAddedJobSource)
22
    IRevisionMailJobSource, IRevisionsAddedJobSource)
3768
@@ -32,12 +32,12 @@
3769
32
        jobs = list(getUtility(IRevisionMailJobSource).iterReady())
32
        jobs = list(getUtility(IRevisionMailJobSource).iterReady())
3770
33
        jobs.extend(getUtility(IRevisionsAddedJobSource).iterReady())
33
        jobs.extend(getUtility(IRevisionsAddedJobSource).iterReady())
3771
34
        runner = JobRunner(jobs, self.logger)
34
        runner = JobRunner(jobs, self.logger)
3774
35
        server = get_scanner_server()
35
        server = get_ro_server()
3775
36
        server.setUp()
36
        server.start_server()
3776
37
        try:
37
        try:
3777
38
            runner.runAll()
38
            runner.runAll()
3778
39
        finally:
39
        finally:
3780
40
            server.tearDown()
40
            server.stop_server()
3781
41
        self.logger.info(
41
        self.logger.info(
3782
42
            'Ran %d RevisionMailJobs.' % len(runner.completed_jobs))
42
            'Ran %d RevisionMailJobs.' % len(runner.completed_jobs))
3783
43
43
3784
44
44
3785
=== modified file 'cronscripts/supermirror-pull.py'
3786
--- cronscripts/supermirror-pull.py	2009-09-29 00:26:59 +0000
3787
+++ cronscripts/supermirror-pull.py	2010-05-05 01:58:48 +0000
3788
@@ -1,4 +1,4 @@
3790
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3791
2
#
2
#
3792
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3793
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3794
@@ -11,10 +11,10 @@
3795
11
from twisted.internet import defer, reactor
11
from twisted.internet import defer, reactor
3796
12
from twisted.python import log as tplog
12
from twisted.python import log as tplog
3797
13
13
3798
14
from lp.codehosting.puller import mirror, scheduler
3799
15
from canonical.config import config
14
from canonical.config import config
3800
16
from canonical.launchpad.scripts import logger_options
15
from canonical.launchpad.scripts import logger_options
3802
17
from canonical.twistedsupport.loggingsupport import (
16
from lp.codehosting.puller import mirror, scheduler
3803
17
from lp.services.twistedsupport.loggingsupport import (
3804
18
    LoggingProxy, set_up_logging_for_script)
18
    LoggingProxy, set_up_logging_for_script)
3805
19
19
3806
20
def clean_shutdown(ignored):
20
def clean_shutdown(ignored):
3807
@@ -38,12 +38,14 @@
3808
38
if __name__ == '__main__':
38
if __name__ == '__main__':
3809
39
    parser = OptionParser()
39
    parser = OptionParser()
3810
40
    logger_options(parser)
40
    logger_options(parser)
3811
41
    parser.add_option('--branch-type', action='append', default=[])
3812
41
    (options, arguments) = parser.parse_args()
42
    (options, arguments) = parser.parse_args()
3813
42
    if arguments:
43
    if arguments:
3814
43
        parser.error("Unhandled arguments %s" % repr(arguments))
44
        parser.error("Unhandled arguments %s" % repr(arguments))
3815
44
    log = set_up_logging_for_script(options, 'supermirror_puller')
45
    log = set_up_logging_for_script(options, 'supermirror_puller')
3816
45
    manager = scheduler.JobScheduler(
46
    manager = scheduler.JobScheduler(
3818
46
        LoggingProxy(config.codehosting.branch_puller_endpoint, log), log)
47
        LoggingProxy(config.codehosting.codehosting_endpoint, log), log,
3819
48
        options.branch_type)
3820
47
49
3821
48
    reactor.callWhenRunning(run_mirror, log, manager)
50
    reactor.callWhenRunning(run_mirror, log, manager)
3822
49
    reactor.run()
51
    reactor.run()
3823
50
52
3824
=== modified file 'cronscripts/translations-export-to-branch.py'
3825
--- cronscripts/translations-export-to-branch.py	2009-07-17 18:46:25 +0000
3826
+++ cronscripts/translations-export-to-branch.py	2010-05-05 01:58:48 +0000
3827
@@ -1,4 +1,4 @@
3829
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3830
2
# pylint: disable-msg=W0403
2
# pylint: disable-msg=W0403
3831
3
#
3
#
3832
4
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3833
5
5
3834
=== modified file 'cronscripts/update-bugtask-targetnamecaches.py'
3835
--- cronscripts/update-bugtask-targetnamecaches.py	2009-06-24 20:52:01 +0000
3836
+++ cronscripts/update-bugtask-targetnamecaches.py	2010-05-05 01:58:48 +0000
3837
@@ -1,4 +1,4 @@
3839
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3840
2
#
2
#
3841
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3842
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3843
5
5
3844
=== modified file 'cronscripts/update-cve.py'
3845
--- cronscripts/update-cve.py	2009-06-24 20:52:01 +0000
3846
+++ cronscripts/update-cve.py	2010-05-05 01:58:48 +0000
3847
@@ -1,4 +1,4 @@
3849
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3850
2
#
2
#
3851
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3852
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3853
5
5
3854
=== modified file 'cronscripts/update-debwatches.py'
3855
--- cronscripts/update-debwatches.py	2009-06-24 20:52:01 +0000
3856
+++ cronscripts/update-debwatches.py	2010-05-05 01:58:48 +0000
3857
@@ -1,4 +1,4 @@
3859
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3860
2
#
2
#
3861
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3862
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3863
@@ -6,11 +6,11 @@
3864
6
# This script runs through the set of Debbugs watches, and tries to
6
# This script runs through the set of Debbugs watches, and tries to
3865
7
# syncronise each of those to the malone bug which is watching it.
7
# syncronise each of those to the malone bug which is watching it.
3866
8
8
3867
9
import _pythonpath
3868
9
import os
10
import os
3869
10
import sys
11
import sys
3870
11
import email
12
import email
3871
12
import logging
13
import logging
3872
13
import _pythonpath
3873
14
14
3874
15
# zope bits
15
# zope bits
3875
16
from zope.component import getUtility
16
from zope.component import getUtility
3876
17
17
3877
=== modified file 'cronscripts/update-pkgcache.py'
3878
--- cronscripts/update-pkgcache.py	2009-06-24 20:52:01 +0000
3879
+++ cronscripts/update-pkgcache.py	2010-05-05 01:58:48 +0000
3880
@@ -1,4 +1,4 @@
3882
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3883
2
#
2
#
3884
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3885
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3886
@@ -47,7 +47,7 @@
3887
47
        PPA archives caches are consolidated in a Archive row to optimize
47
        PPA archives caches are consolidated in a Archive row to optimize
3888
48
        searches across PPAs.
48
        searches across PPAs.
3889
49
        """
49
        """
3891
50
        for distroseries in distribution.serieses:
50
        for distroseries in distribution.series:
3892
51
            self.updateDistroSeriesCache(distroseries, archive)
51
            self.updateDistroSeriesCache(distroseries, archive)
3893
52
52
3894
53
        distribution.removeOldCacheItems(archive, log=self.logger)
53
        distribution.removeOldCacheItems(archive, log=self.logger)
3895
@@ -101,6 +101,6 @@
3896
101
101
3897
102
if __name__ == '__main__':
102
if __name__ == '__main__':
3898
103
    script = PackageCacheUpdater(
103
    script = PackageCacheUpdater(
3900
104
        'update-cache', dbuser=config.statistician.dbuser)
104
        'update-cache', dbuser="update-pkg-cache")
3901
105
    script.lock_and_run()
105
    script.lock_and_run()
3902
106
106
3903
107
107
3904
=== modified file 'cronscripts/update-remote-product.py'
3905
--- cronscripts/update-remote-product.py	2009-06-24 20:52:01 +0000
3906
+++ cronscripts/update-remote-product.py	2010-05-05 01:58:48 +0000
3907
@@ -1,14 +1,14 @@
3909
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3910
2
#
2
#
3911
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3912
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3913
5
5
3914
6
"""Cron job to update Product.remote_product using bug watch information.  """
3915
7
3916
6
# pylint: disable-msg=W0403
8
# pylint: disable-msg=W0403
3919
7
9
import _pythonpath
3918
8
"""Cron job to update Product.remote_product using bug watch information.  """
3920
9
10
3921
10
import time
11
import time
3922
11
import _pythonpath
3923
12
12
3924
13
from canonical.config import config
13
from canonical.config import config
3925
14
from lp.services.scripts.base import LaunchpadCronScript
14
from lp.services.scripts.base import LaunchpadCronScript
3926
15
15
3927
=== modified file 'cronscripts/update-sourceforge-remote-products.py'
3928
--- cronscripts/update-sourceforge-remote-products.py	2009-06-24 20:52:01 +0000
3929
+++ cronscripts/update-sourceforge-remote-products.py	2010-05-05 01:58:48 +0000
3930
@@ -1,14 +1,14 @@
3932
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3933
2
#
2
#
3934
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3935
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3936
5
5
3937
6
"""Cron job to update remote_products using SourceForge project data."""
3938
7
3939
6
# pylint: disable-msg=W0403
8
# pylint: disable-msg=W0403
3942
7
9
import _pythonpath
3941
8
"""Cron job to update remote_products using SourceForge project data."""
3943
9
10
3944
10
import time
11
import time
3945
11
import _pythonpath
3946
12
12
3947
13
from canonical.config import config
13
from canonical.config import config
3948
14
from lp.services.scripts.base import LaunchpadCronScript
14
from lp.services.scripts.base import LaunchpadCronScript
3949
15
15
3950
=== modified file 'cronscripts/update-standing.py'
3951
--- cronscripts/update-standing.py	2009-06-24 20:52:01 +0000
3952
+++ cronscripts/update-standing.py	2010-05-05 01:58:48 +0000
3953
@@ -1,4 +1,4 @@
3955
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3956
2
#
2
#
3957
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3958
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3959
5
5
3960
=== modified file 'cronscripts/update-stats.py'
3961
--- cronscripts/update-stats.py	2009-06-24 20:52:01 +0000
3962
+++ cronscripts/update-stats.py	2010-05-05 01:58:48 +0000
3963
@@ -1,4 +1,4 @@
3965
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
3966
2
#
2
#
3967
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3968
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3969
@@ -28,7 +28,7 @@
3970
28
        # objects are responsible for committing.
28
        # objects are responsible for committing.
3971
29
        distroset = getUtility(IDistributionSet)
29
        distroset = getUtility(IDistributionSet)
3972
30
        for distro in distroset:
30
        for distro in distroset:
3974
31
            for distroseries in distro.serieses:
31
            for distroseries in distro.series:
3975
32
                distroseries.updateStatistics(self.txn)
32
                distroseries.updateStatistics(self.txn)
3976
33
33
3977
34
        launchpad_stats = getUtility(ILaunchpadStatisticSet)
34
        launchpad_stats = getUtility(ILaunchpadStatisticSet)
3978
35
35
3979
=== removed file 'cronscripts/update_preview_diffs.py'
3980
--- cronscripts/update_preview_diffs.py	2009-09-01 19:00:46 +0000
3981
+++ cronscripts/update_preview_diffs.py	1970-01-01 00:00:00 +0000
3982
@@ -1,34 +0,0 @@
3983
1
#!/usr/bin/python2.4
3984
2
#
3985
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3986
4
# GNU Affero General Public License version 3 (see the file LICENSE).
3987
5
3988
6
# pylint: disable-msg=W0403
3989
7
3990
8
"""Update or create previews diffs for branch merge proposals."""
3991
9
3992
10
__metaclass__ = type
3993
11
3994
12
import _pythonpath
3995
13
3996
14
from lp.codehosting.vfs import get_scanner_server
3997
15
from lp.services.job.runner import JobCronScript
3998
16
from lp.code.interfaces.branchmergeproposal import (
3999
17
    IUpdatePreviewDiffJobSource,)
4000
18
4001
19
4002
20
class RunUpdatePreviewDiffJobs(JobCronScript):
4003
21
    """Run UpdatePreviewDiff jobs."""
4004
22
4005
23
    config_name = 'update_preview_diffs'
4006
24
    source_interface = IUpdatePreviewDiffJobSource
4007
25
4008
26
    def setUp(self):
4009
27
        server = get_scanner_server()
4010
28
        server.setUp()
4011
29
        return [server.tearDown]
4012
30
4013
31
4014
32
if __name__ == '__main__':
4015
33
    script = RunUpdatePreviewDiffJobs()
4016
34
    script.lock_and_run()
4017
35
0
4018
=== added file 'cronscripts/upgrade_branches.py'
4019
--- cronscripts/upgrade_branches.py	1970-01-01 00:00:00 +0000
4020
+++ cronscripts/upgrade_branches.py	2010-05-05 01:58:48 +0000
4021
@@ -0,0 +1,25 @@
4022
1
#!/usr/bin/python2.5 -S
4023
2
#
4024
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4025
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4026
5
4027
6
"""Upgrade branches to the most recent format."""
4028
7
4029
8
__metaclass__ = type
4030
9
4031
10
import _pythonpath
4032
11
4033
12
from lp.services.job.runner import JobCronScript
4034
13
from lp.code.interfaces.branchjob import IBranchUpgradeJobSource
4035
14
4036
15
4037
16
class RunUpgradeBranches(JobCronScript):
4038
17
    """Run UpgradeBranchJob jobs."""
4039
18
4040
19
    config_name = 'upgrade_branches'
4041
20
    source_interface = IBranchUpgradeJobSource
4042
21
4043
22
4044
23
if __name__ == '__main__':
4045
24
    script = RunUpgradeBranches()
4046
25
    script.lock_and_run()
4047
0
26
4048
=== added symlink 'daemons/_pythonpath.py'
4049
=== target is u'../_pythonpath.py'
4050
=== removed file 'daemons/buildd-sequencer.tac'
4051
--- daemons/buildd-sequencer.tac	2009-06-24 20:55:31 +0000
4052
+++ daemons/buildd-sequencer.tac	1970-01-01 00:00:00 +0000
4053
@@ -1,31 +0,0 @@
4054
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4055
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4056
3
4057
4
# Twisted Application Configuration file.
4058
5
# Use with "twistd -y <file.tac>", e.g. "twistd -noy server.tac"
4059
6
4060
7
from twisted.application import service
4061
8
4062
9
from canonical.buildd.sequencer import BuildSequencer
4063
10
4064
11
# Construct the application
4065
12
application = service.Application("BuildSequencer")
4066
13
4067
14
class BuildSequencerService(service.Service):
4068
15
    def __init__(self, buildSequencer):
4069
16
        self.buildSequencer = buildSequencer
4070
17
 
4071
18
    def startService(self):
4072
19
        # Kick everything off...
4073
20
        self.buildSequencer.scheduleCallback()
4074
21
 
4075
22
4076
23
# Construct the sequencer. It will automatically schedule the first job.
4077
24
bseq = BuildSequencer()
4078
25
# Make a service out of the sequencer
4079
26
bserv = BuildSequencerService(bseq)
4080
27
4081
28
# Activate the service
4082
29
BuildSequencerService(bseq).setServiceParent(application)
4083
30
4084
31
# Falling off the end here passes into twisted's reactor.
4085
32
0
4086
=== modified file 'daemons/buildd-slave.tac'
4087
--- daemons/buildd-slave.tac	2009-06-24 20:55:31 +0000
4088
+++ daemons/buildd-slave.tac	2010-05-05 01:58:48 +0000
4089
@@ -1,4 +1,4 @@
4091
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
1
# Copyright 2009, 2010 Canonical Ltd.  This software is licensed under the
4092
2
# GNU Affero General Public License version 3 (see the file LICENSE).
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4093
3
3
4094
4
# Author: Daniel Silverstone <daniel.silverstone@canonical.com>
4
# Author: Daniel Silverstone <daniel.silverstone@canonical.com>
4095
@@ -8,7 +8,11 @@
4096
8
# passed through to the twistd log too. this could get dangerous/big
8
# passed through to the twistd log too. this could get dangerous/big
4097
9
9
4098
10
from twisted.application import service, strports
10
from twisted.application import service, strports
4100
11
from canonical.buildd import XMLRPCBuildDSlave, DebianBuildManager
11
from canonical.buildd import XMLRPCBuildDSlave
4101
12
from canonical.buildd.binarypackage import BinaryPackageBuildManager
4102
13
from canonical.buildd.sourcepackagerecipe import (
4103
14
    SourcePackageRecipeBuildManager)
4104
15
from canonical.buildd.translationtemplates import TranslationTemplatesBuildManager
4105
12
from canonical.launchpad.daemons import tachandler
16
from canonical.launchpad.daemons import tachandler
4106
13
17
4107
14
from twisted.web import server, resource, static
18
from twisted.web import server, resource, static
4108
@@ -22,7 +26,11 @@
4109
22
conf.read(conffile)
26
conf.read(conffile)
4110
23
slave = XMLRPCBuildDSlave(conf)
27
slave = XMLRPCBuildDSlave(conf)
4111
24
28
4113
25
slave.registerBuilder(DebianBuildManager,"debian")
29
# 'debian' is the old name. It remains here for compatibility.
4114
30
slave.registerBuilder(BinaryPackageBuildManager, "debian")
4115
31
slave.registerBuilder(BinaryPackageBuildManager, "binarypackage")
4116
32
slave.registerBuilder(SourcePackageRecipeBuildManager, "sourcepackagerecipe")
4117
33
slave.registerBuilder(TranslationTemplatesBuildManager, 'translation-templates')
4118
26
34
4119
27
application = service.Application('BuildDSlave')
35
application = service.Application('BuildDSlave')
4120
28
builddslaveService = service.IServiceCollection(application)
36
builddslaveService = service.IServiceCollection(application)
4121
@@ -43,5 +51,5 @@
4122
43
#
51
#
4123
44
# python
52
# python
4124
45
# import xmlrpclib
53
# import xmlrpclib
4126
46
# s = xmlrpclib.Server("http://localhost:8221/")
54
# s = xmlrpclib.ServerProxy("http://localhost:8221/rpc")
4127
47
# s.echo("Hello World")
55
# s.echo("Hello World")
4128
48
56
4129
=== added file 'daemons/cache-database-replication-lag.py'
4130
--- daemons/cache-database-replication-lag.py	1970-01-01 00:00:00 +0000
4131
+++ daemons/cache-database-replication-lag.py	2010-05-05 01:58:48 +0000
4132
@@ -0,0 +1,53 @@
4133
1
#!/usr/bin/python2.5 -S
4134
2
#
4135
3
# Copyright 2010 Canonical Ltd.  This software is licensed under the
4136
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4137
5
4138
6
"""Calculate database replication lag and cache it."""
4139
7
4140
8
__metaclass__ = type
4141
9
__all__ = []
4142
10
4143
11
import _pythonpath
4144
12
4145
13
import sys
4146
14
import time
4147
15
4148
16
import psycopg2
4149
17
4150
18
from canonical.database.sqlbase import connect, ISOLATION_LEVEL_AUTOCOMMIT
4151
19
from canonical.launchpad.scripts import db_options, logger
4152
20
from lp.scripts.helpers import LPOptionParser
4153
21
4154
22
4155
23
def main(args=None):
4156
24
    parser = LPOptionParser()
4157
25
    db_options(parser)
4158
26
    parser.add_option(
4159
27
        "-s", "--sleep", dest="sleep", type="int", default=5,
4160
28
        metavar="SECS", help="Wait SECS seconds between refreshes.")
4161
29
4162
30
    (options, args) = parser.parse_args(args)
4163
31
    if len(args) != 0:
4164
32
        parser.error("Too many arguments.")
4165
33
4166
34
    log = logger(options)
4167
35
4168
36
    while True:
4169
37
        try:
4170
38
            con = connect(user="lagmon", isolation=ISOLATION_LEVEL_AUTOCOMMIT)
4171
39
            cur = con.cursor()
4172
40
            while True:
4173
41
                cur.execute("SELECT update_replication_lag_cache()")
4174
42
                if cur.fetchone()[0]:
4175
43
                    log.info("Updated.")
4176
44
                else:
4177
45
                    log.error("update_replication_lag_cache() failed.")
4178
46
                time.sleep(options.sleep)
4179
47
        except psycopg2.Error, x:
4180
48
            log.error("%s. Retrying.", str(x).strip())
4181
49
            time.sleep(options.sleep)
4182
50
4183
51
4184
52
if __name__ == '__main__':
4185
53
    sys.exit(main())
4186
0
54
4187
=== modified file 'daemons/librarian.tac'
4188
--- daemons/librarian.tac	2009-06-24 20:55:31 +0000
4189
+++ daemons/librarian.tac	2010-05-05 01:58:48 +0000
4190
@@ -4,6 +4,10 @@
4191
4
# Twisted Application Configuration file.
4
# Twisted Application Configuration file.
4192
5
# Use with "twistd2.4 -y <file.tac>", e.g. "twistd -noy server.tac"
5
# Use with "twistd2.4 -y <file.tac>", e.g. "twistd -noy server.tac"
4193
6
6
4194
7
import signal
4195
8
4196
9
from meliae import scanner
4197
10
4198
7
from twisted.application import service, strports
11
from twisted.application import service, strports
4199
8
from twisted.web import server
12
from twisted.web import server
4200
9
13
4201
@@ -11,6 +15,7 @@
4202
11
from canonical.launchpad.daemons import tachandler
15
from canonical.launchpad.daemons import tachandler
4203
12
from canonical.launchpad.scripts import execute_zcml_for_scripts
16
from canonical.launchpad.scripts import execute_zcml_for_scripts
4204
13
17
4205
18
from canonical.librarian.interfaces import DUMP_FILE, SIGDUMPMEM
4206
14
from canonical.librarian.libraryprotocol import FileUploadFactory
19
from canonical.librarian.libraryprotocol import FileUploadFactory
4207
15
from canonical.librarian import storage, db
20
from canonical.librarian import storage, db
4208
16
from canonical.librarian import web as fatweb
21
from canonical.librarian import web as fatweb
4209
@@ -64,3 +69,9 @@
4210
64
webPort = config.librarian.restricted_download_port
69
webPort = config.librarian.restricted_download_port
4211
65
uploadPort = config.librarian.restricted_upload_port
70
uploadPort = config.librarian.restricted_upload_port
4212
66
setUpListener(uploadPort, webPort, restricted=True)
71
setUpListener(uploadPort, webPort, restricted=True)
4213
72
4214
73
# Setup a signal handler to dump the process' memory upon 'kill -44'.
4215
74
def sigdumpmem_handler(signum, frame):
4216
75
    scanner.dump_all_objects(DUMP_FILE)
4217
76
4218
77
signal.signal(SIGDUMPMEM, sigdumpmem_handler)
4219
67
78
4220
=== modified file 'daemons/poppy-upload.py'
4221
--- daemons/poppy-upload.py	2009-06-30 21:26:26 +0000
4222
+++ daemons/poppy-upload.py	2010-05-05 01:58:48 +0000
4223
@@ -1,57 +1,10 @@
4225
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5
4226
2
#
2
#
4227
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4228
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4229
5
5
4230
6
import sys
6
import sys
4279
7
import logging
7
from lp.poppy.daemon import main
4232
8
import optparse
4233
9
4234
10
from canonical.poppy.server import run_server
4235
11
from lp.archiveuploader.poppyinterface import PoppyInterface
4236
12
from canonical.launchpad.scripts import logger, logger_options
4237
13
4238
14
4239
15
def main():
4240
16
4241
17
    parser = optparse.OptionParser()
4242
18
    logger_options(parser)
4243
19
4244
20
    parser.add_option("--cmd", action="store", metavar="CMD",
4245
21
                      help="Run CMD after each upload completion")
4246
22
4247
23
    parser.add_option("--allow-user", action="store", metavar="USER",
4248
24
                      default='ubuntu',
4249
25
                      help="Username allowed to log in.")
4250
26
4251
27
    parser.add_option("--permissions", action="store", metavar="PERMS",
4252
28
                      default='g+rwxs',
4253
29
                      help="Permissions to chmod the targetfsroot with "
4254
30
                      "before letting go of the directory.")
4255
31
4256
32
    options, args = parser.parse_args()
4257
33
4258
34
    log = logger(options, "poppy-upload")
4259
35
4260
36
    if len(args) != 2:
4261
37
        print "usage: poppy-upload.py rootuploaddirectory port"
4262
38
        return 1
4263
39
4264
40
    root, port = args
4265
41
    # host = "127.0.0.1"
4266
42
    # host = "82.211.81.167" # Drescher's public IP
4267
43
    host = "0.0.0.0"
4268
44
    ident = "lucille upload server"
4269
45
    numthreads = 4
4270
46
4271
47
    iface = PoppyInterface(root, log, allow_user=options.allow_user,
4272
48
                           cmd=options.cmd,
4273
49
                           perms=options.permissions)
4274
50
4275
51
    run_server(host, int(port), ident, numthreads,
4276
52
               iface.new_client_hook, iface.client_done_hook,
4277
53
               iface.auth_verify_hook)
4278
54
    return 0
4280
55
8
4281
56
if __name__ == '__main__':
9
if __name__ == '__main__':
4282
57
    sys.exit(main())
10
    sys.exit(main())
4283
58
11
4284
=== modified file 'daemons/sftp.tac'
4285
--- daemons/sftp.tac	2009-06-24 20:55:31 +0000
4286
+++ daemons/sftp.tac	2010-05-05 01:58:48 +0000
4287
@@ -1,4 +1,4 @@
4289
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
1
# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
4290
2
# GNU Affero General Public License version 3 (see the file LICENSE).
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4291
3
3
4292
4
# This is a Twisted application config file.  To run, use:
4
# This is a Twisted application config file.  To run, use:
4293
@@ -7,13 +7,28 @@
4294
7
7
4295
8
from twisted.application import service
8
from twisted.application import service
4296
9
9
4297
10
from canonical.config import config
4298
10
from canonical.launchpad.daemons import tachandler
11
from canonical.launchpad.daemons import tachandler
4303
11
from lp.codehosting.sshserver.service import SSHService
12
4304
12
13
from lp.codehosting.sshserver.daemon import (
4305
13
14
    ACCESS_LOG_NAME, get_key_path, LOG_NAME, make_portal, OOPS_CONFIG_SECTION,
4306
14
# Construct an Application that includes a supermirror SFTP service. 
15
    PRIVATE_KEY_FILE, PUBLIC_KEY_FILE)
4307
16
from lp.services.sshserver.service import SSHService
4308
17
4309
18
4310
19
# Construct an Application that has the codehosting SSH server.
4311
15
application = service.Application('sftponly')
20
application = service.Application('sftponly')
4313
16
svc = SSHService()
21
svc = SSHService(
4314
22
    portal=make_portal(),
4315
23
    private_key_path=get_key_path(PRIVATE_KEY_FILE),
4316
24
    public_key_path=get_key_path(PUBLIC_KEY_FILE),
4317
25
    oops_configuration=OOPS_CONFIG_SECTION,
4318
26
    main_log=LOG_NAME,
4319
27
    access_log=ACCESS_LOG_NAME,
4320
28
    access_log_path=config.codehosting.access_log,
4321
29
    strport=config.codehosting.port,
4322
30
    idle_timeout=config.codehosting.idle_timeout,
4323
31
    banner=config.codehosting.banner)
4324
17
svc.setServiceParent(application)
32
svc.setServiceParent(application)
4325
18
33
4326
19
# Service that announces when the daemon is ready
34
# Service that announces when the daemon is ready
4327
20
35
4328
=== modified file 'database/replication/Makefile'
4329
--- database/replication/Makefile	2009-06-24 21:17:33 +0000
4330
+++ database/replication/Makefile	2010-05-05 01:58:48 +0000
4331
@@ -18,6 +18,11 @@
4332
18
#  $ make stagingsetup STAGING_CONFIG=dev-staging STAGING_DUMP=launchpad.dump
18
#  $ make stagingsetup STAGING_CONFIG=dev-staging STAGING_DUMP=launchpad.dump
4333
19
#  $ make stagingswitch STAGING_CONFIG=dev-staging
19
#  $ make stagingswitch STAGING_CONFIG=dev-staging
4334
20
#
20
#
4335
21
# To restore a dogfood database:
4336
22
#
4337
23
#  $ cd database/replication
4338
24
#  $ make dogfood DOGFOOD_DBNAME=launchpad_dogfood DOGFOOD_DUMP=launchpad.dump
4339
25
#
4340
21
26
4341
22
# This used to be 10 seconds, so we always ran staging lagged to detect
27
# This used to be 10 seconds, so we always ran staging lagged to detect
4342
23
# replication glitches more easily. However, this does not play well
28
# replication glitches more easily. However, this does not play well
4343
@@ -31,12 +36,24 @@
4344
31
STAGING_CONFIG=staging # For swapping fresh db into place.
36
STAGING_CONFIG=staging # For swapping fresh db into place.
4345
32
STAGING_DUMP=launchpad.dump # Dumpfile to build new staging from.
37
STAGING_DUMP=launchpad.dump # Dumpfile to build new staging from.
4346
33
STAGING_TABLESPACE=pg_default # 'pg_default' for default
38
STAGING_TABLESPACE=pg_default # 'pg_default' for default
4347
39
DOGFOOD_DBNAME=launchpad_dogfood
4348
40
DOGFOOD_DUMP=launchpad.dump
4349
34
41
4350
35
_CONFIG=overridden-on-command-line
42
_CONFIG=overridden-on-command-line
4351
36
_SLAVE_TABLESPACE=pg_default
43
_SLAVE_TABLESPACE=pg_default
4352
37
44
4353
38
PGMASSACRE=../../utilities/pgmassacre.py
45
PGMASSACRE=../../utilities/pgmassacre.py
4354
39
46
4355
47
CREATEDB_83=createdb --encoding=UTF8
4356
48
CREATEDB_84=createdb --encoding=UTF8 --locale=C --template=template0
4357
49
CREATEDB=${CREATEDB_83}
4358
50
4359
51
# Turn off output silencing so we can see details of staging deployments.
4360
52
# Without the timestamps, we are unable to estimate production deployment
4361
53
# times.
4362
54
#SHHH=../../utilities/shhh.py
4363
55
SHHH=
4364
56
4365
40
default:
57
default:
4366
41
	echo Usage: make [start|stop|restart]
58
	echo Usage: make [start|stop|restart]
4367
42
59
4368
@@ -61,7 +78,7 @@
4369
61
	
78
	
4370
62
	# Replicate it again, so we can test with multiple slaves.
79
	# Replicate it again, so we can test with multiple slaves.
4371
63
	-${PGMASSACRE} launchpad_dev_slave2
80
	-${PGMASSACRE} launchpad_dev_slave2
4373
64
	createdb --encoding=UTF8 launchpad_dev_slave2
81
	${CREATEDB} launchpad_dev_slave2
4374
65
	LPCONFIG=${DEV_CONFIG} ./slon_ctl.py start \
82
	LPCONFIG=${DEV_CONFIG} ./slon_ctl.py start \
4375
66
		 node3_node 'dbname=launchpad_dev_slave2 user=slony'
83
		 node3_node 'dbname=launchpad_dev_slave2 user=slony'
4376
67
	LPCONFIG=${DEV_CONFIG} ./new-slave.py 3 launchpad_dev_slave2
84
	LPCONFIG=${DEV_CONFIG} ./new-slave.py 3 launchpad_dev_slave2
4377
@@ -81,13 +98,12 @@
4378
81
	    _MASTER=lpmain_staging_new _SLAVE=lpmain_staging_slave_new \
98
	    _MASTER=lpmain_staging_new _SLAVE=lpmain_staging_slave_new \
4379
82
	    LAG="0 seconds"
99
	    LAG="0 seconds"
4380
83
	# Create the DB with the desired default tablespace.
100
	# Create the DB with the desired default tablespace.
4386
84
	createdb --encoding UTF8 --tablespace ${STAGING_TABLESPACE} \
101
	${CREATEDB} --tablespace ${STAGING_TABLESPACE} lpmain_staging_new
4387
85
	    lpmain_staging_new
102
	# Restore the database. We need to restore permissions, despite
4388
86
	# Restore the DB schema. Don't restore permissions - it will blow
103
	# later running security.py, to pull in permissions granted on
4389
87
	# up when roles don't exist in this cluster, and we rebuild it later
104
	# production to users not maintained by security.py.
4385
88
	# with security.py anyway.
4390
89
	pg_restore --dbname=lpmain_staging_new \
105
	pg_restore --dbname=lpmain_staging_new \
4392
90
	    --no-acl --exit-on-error ${STAGING_DUMP}
106
	    --no-owner --exit-on-error ${STAGING_DUMP}
4393
91
	# Uninstall Slony-I if it is installed - a pg_dump of a DB with
107
	# Uninstall Slony-I if it is installed - a pg_dump of a DB with
4394
92
	# Slony-I installed isn't usable without this step.
108
	# Slony-I installed isn't usable without this step.
4395
93
	LPCONFIG=${NEW_STAGING_CONFIG} ./repair-restored-db.py
109
	LPCONFIG=${NEW_STAGING_CONFIG} ./repair-restored-db.py
4396
@@ -118,6 +134,15 @@
4397
118
	# Start the slon daemons, with requested lag.
134
	# Start the slon daemons, with requested lag.
4398
119
	LPCONFIG=${STAGING_CONFIG} ./slon_ctl.py --lag="${LAG}" start
135
	LPCONFIG=${STAGING_CONFIG} ./slon_ctl.py --lag="${LAG}" start
4399
120
136
4400
137
dogfood:
4401
138
	${CREATEDB} ${DOGFOOD_DBNAME}
4402
139
	pg_restore --dbname=${DOGFOOD_DBNAME} --no-acl --no-owner \
4403
140
	    --exit-on-error ${DOGFOOD_DUMP}
4404
141
	./repair-restored-db.py -d ${DOGFOOD_DBNAME}
4405
142
	../schema/upgrade.py -d ${DOGFOOD_DBNAME}
4406
143
	../schema/fti.py -d ${DOGFOOD_DBNAME}
4407
144
	../schema/security.py -d ${DOGFOOD_DBNAME}
4408
145
4409
121
_prelim:
146
_prelim:
4410
122
	@echo LPCONFIG currently ${LPCONFIG}
147
	@echo LPCONFIG currently ${LPCONFIG}
4411
123
	# Create the slony PostgreSQL superuser if necessary.
148
	# Create the slony PostgreSQL superuser if necessary.
4412
@@ -132,25 +157,26 @@
4413
132
_replicate:
157
_replicate:
4414
133
	@echo LPCONFIG currently ${LPCONFIG}
158
	@echo LPCONFIG currently ${LPCONFIG}
4415
134
	# Start the slon daemon for the master.
159
	# Start the slon daemon for the master.
4417
135
	./slon_ctl.py start \
160
	./slon_ctl.py --lag="0 seconds" start \
4418
136
		 node1_node "dbname=${_MASTER} user=slony"
161
		 node1_node "dbname=${_MASTER} user=slony"
4419
137
	# Initialize the cluster and create replication sets.
162
	# Initialize the cluster and create replication sets.
4420
138
	./initialize.py
163
	./initialize.py
4421
139
	# Create the soon-to-be-slave database, empty at this point.
164
	# Create the soon-to-be-slave database, empty at this point.
4423
140
	createdb --encoding=UTF8 --tablespace=${_SLAVE_TABLESPACE} ${_SLAVE}
165
	${CREATEDB} --tablespace=${_SLAVE_TABLESPACE} ${_SLAVE}
4424
141
	# Start the slon daemon for the slave
166
	# Start the slon daemon for the slave
4426
142
	./slon_ctl.py start node2_node "dbname=${_SLAVE} user=slony"
167
	./slon_ctl.py --lag="0 seconds" start \
4427
168
	    node2_node "dbname=${_SLAVE} user=slony"
4428
143
	# Setup the slave
169
	# Setup the slave
4429
144
	./new-slave.py 2 "dbname=${_SLAVE}"
170
	./new-slave.py 2 "dbname=${_SLAVE}"
4430
145
	# Upgrade all databases in the cluster and reset security.
171
	# Upgrade all databases in the cluster and reset security.
4437
146
	../schema/upgrade.py
172
	@echo Running upgrade.py `date`
4438
147
	../schema/fti.py
173
	${SHHH} ../schema/upgrade.py
4439
148
	../schema/security.py --cluster -U slony
174
	@echo Running fti.py `date`
4440
149
	# Migrate tables to the authdb replication set, creating the set
175
	${SHHH} ../schema/fti.py
4441
150
	# and subscribing nodes to it as necessary.
176
	@echo Running security.py `date`
4442
151
	./populate_auth_replication_set.py -U slony
177
	./slon_ctl.py stop # security.py can deadlock with slony
4443
178
	${SHHH} ../schema/security.py --cluster -U slony
4444
152
	# Restart slon daemons with default lag setting.
179
	# Restart slon daemons with default lag setting.
4445
153
	./slon_ctl.py stop
4446
154
	./slon_ctl.py --lag="${LAG}" start
180
	./slon_ctl.py --lag="${LAG}" start
4447
155
	# Generate a preamble for manual slonik(1) usage.
181
	# Generate a preamble for manual slonik(1) usage.
4448
156
	./preamble.py > preamble.sk
182
	./preamble.py > preamble.sk
4449
157
183
4450
=== modified file 'database/replication/helpers.py'
4451
--- database/replication/helpers.py	2009-06-24 21:17:33 +0000
4452
+++ database/replication/helpers.py	2010-05-05 01:58:48 +0000
4453
@@ -26,19 +26,20 @@
4454
26
# The namespace in the database used to contain all the Slony-I tables.
26
# The namespace in the database used to contain all the Slony-I tables.
4455
27
CLUSTER_NAMESPACE = '_%s' % CLUSTERNAME
27
CLUSTER_NAMESPACE = '_%s' % CLUSTERNAME
4456
28
28
4458
29
# Seed tables for the authdb replication set to be passed to
29
# Replication set id constants. Don't change these without DBA help.
4459
30
LPMAIN_SET_ID = 1
4460
31
HOLDING_SET_ID = 666
4461
32
LPMIRROR_SET_ID = 4
4462
33
4463
34
# Seed tables for the lpmain replication set to be passed to
4464
30
# calculate_replication_set().
35
# calculate_replication_set().
4466
31
AUTHDB_SEED = frozenset([
36
LPMAIN_SEED = frozenset([
4467
32
    ('public', 'account'),
37
    ('public', 'account'),
4468
38
    ('public', 'openidnonce'),
4469
33
    ('public', 'openidassociation'),
39
    ('public', 'openidassociation'),
4470
34
    ('public', 'openidnonce'),
4471
35
    ])
4472
36
4473
37
# Seed tables for the lpmain replication set to be passed to
4474
38
# calculate_replication_set().
4475
39
LPMAIN_SEED = frozenset([
4476
40
    ('public', 'person'),
40
    ('public', 'person'),
4477
41
    ('public', 'launchpaddatabaserevision'),
41
    ('public', 'launchpaddatabaserevision'),
4478
42
    ('public', 'databasereplicationlag'),
4479
42
    ('public', 'fticache'),
43
    ('public', 'fticache'),
4480
43
    ('public', 'nameblacklist'),
44
    ('public', 'nameblacklist'),
4481
44
    ('public', 'openidconsumerassociation'),
45
    ('public', 'openidconsumerassociation'),
4482
@@ -51,14 +52,49 @@
4483
51
    ('public', 'launchpadstatistic'),
52
    ('public', 'launchpadstatistic'),
4484
52
    ('public', 'parsedapachelog'),
53
    ('public', 'parsedapachelog'),
4485
53
    ('public', 'shipitsurvey'),
54
    ('public', 'shipitsurvey'),
4487
54
    ('public', 'openidassociations'), # Remove this in April 2009 or later.
55
    ('public', 'databasereplicationlag'),
4488
55
    ])
56
    ])
4489
56
57
4490
57
# Explicitly list tables that should not be replicated. This includes the
58
# Explicitly list tables that should not be replicated. This includes the
4491
58
# session tables, as these might exist in developer databases but will not
59
# session tables, as these might exist in developer databases but will not
4492
59
# exist in the production launchpad database.
60
# exist in the production launchpad database.
4493
60
IGNORED_TABLES = set([
61
IGNORED_TABLES = set([
4495
61
    'public.secret', 'public.sessiondata', 'public.sessionpkgdata'])
62
    # Session tables that in some situations will exist in the main lp
4496
63
    # database.
4497
64
    'public.secret', 'public.sessiondata', 'public.sessionpkgdata',
4498
65
    # Mirror tables, per Bug #489078. These tables have their own private
4499
66
    # replication set that is setup manually.
4500
67
    'public.lp_account',
4501
68
    'public.lp_person',
4502
69
    'public.lp_personlocation',
4503
70
    'public.lp_teamparticipation',
4504
71
    # Ubuntu SSO database. These tables where created manually by ISD
4505
72
    # and the Launchpad scripts should not mess with them. Eventually
4506
73
    # these tables will be in a totally separate database.
4507
74
    'public.auth_permission',
4508
75
    'public.auth_group',
4509
76
    'public.auth_user',
4510
77
    'public.auth_message',
4511
78
    'public.django_content_type',
4512
79
    'public.auth_permission',
4513
80
    'public.django_session',
4514
81
    'public.django_site',
4515
82
    'public.django_admin_log',
4516
83
    'public.ssoopenidrpconfig',
4517
84
    'public.auth_group_permissions',
4518
85
    'public.auth_user_groups',
4519
86
    'public.auth_user_user_permissions',
4520
87
    'public.oauth_nonce',
4521
88
    'public.oauth_consumer',
4522
89
    'public.oauth_token',
4523
90
    'public.api_user',
4524
91
    'public.oauth_consumer_id_seq',
4525
92
    'public.api_user_id_seq',
4526
93
    'public.oauth_nonce_id_seq',
4527
94
    ])
4528
95
4529
96
# Calculate IGNORED_SEQUENCES
4530
97
IGNORED_SEQUENCES = set('%s_id_seq' % table for table in IGNORED_TABLES)
4531
62
98
4532
63
99
4533
64
def slony_installed(con):
100
def slony_installed(con):
4534
@@ -101,7 +137,7 @@
4535
101
137
4536
102
def sync(timeout):
138
def sync(timeout):
4537
103
    """Generate a sync event and wait for it to complete on all nodes.
139
    """Generate a sync event and wait for it to complete on all nodes.
4539
104
   
140
4540
105
    This means that all pending events have propagated and are in sync
141
    This means that all pending events have propagated and are in sync
4541
106
    to the point in time this method was called. This might take several
142
    to the point in time this method was called. This might take several
4542
107
    hours if there is a large backlog of work to replicate.
143
    hours if there is a large backlog of work to replicate.
4543
@@ -135,12 +171,13 @@
4544
135
        script = preamble() + script
171
        script = preamble() + script
4545
136
172
4546
137
    if sync is not None:
173
    if sync is not None:
4548
138
        script = script + dedent("""\
174
        sync_script = dedent("""\
4549
139
            sync (id = @master_node);
175
            sync (id = @master_node);
4550
140
            wait for event (
176
            wait for event (
4552
141
                origin = ALL, confirmed = ALL,
177
                origin = @master_node, confirmed = ALL,
4553
142
                wait on = @master_node, timeout = %d);
178
                wait on = @master_node, timeout = %d);
4554
143
            """ % sync)
179
            """ % sync)
4555
180
        script = script + sync_script
4556
144
181
4557
145
    # Copy the script to a NamedTemporaryFile rather than just pumping it
182
    # Copy the script to a NamedTemporaryFile rather than just pumping it
4558
146
    # to slonik via stdin. This way it can be examined if slonik appears
183
    # to slonik via stdin. This way it can be examined if slonik appears
4559
@@ -151,7 +188,7 @@
4560
151
188
4561
152
    # Run slonik
189
    # Run slonik
4562
153
    log.debug("Executing slonik script %s" % script_on_disk.name)
190
    log.debug("Executing slonik script %s" % script_on_disk.name)
4564
154
    log.log(DEBUG2, script)
191
    log.log(DEBUG2, 'Running script:\n%s' % script)
4565
155
    returncode = subprocess.call(['slonik', script_on_disk.name])
192
    returncode = subprocess.call(['slonik', script_on_disk.name])
4566
156
193
4567
157
    if returncode != 0:
194
    if returncode != 0:
4568
@@ -255,7 +292,7 @@
4569
255
        assert len(node_ids) == 1, "Multiple nodes but no paths."
292
        assert len(node_ids) == 1, "Multiple nodes but no paths."
4570
256
        master_node_id = node_ids[0]
293
        master_node_id = node_ids[0]
4571
257
        master_connection_string = ConnectionString(
294
        master_connection_string = ConnectionString(
4573
258
            config.database.main_master)
295
            config.database.rw_main_master)
4574
259
        master_connection_string.user = 'slony'
296
        master_connection_string.user = 'slony'
4575
260
        return [Node(
297
        return [Node(
4576
261
            master_node_id, 'node%d_node' % master_node_id,
298
            master_node_id, 'node%d_node' % master_node_id,
4577
@@ -282,10 +319,10 @@
4578
282
        cluster name = sl;
319
        cluster name = sl;
4579
283
320
4580
284
        # Symbolic ids for replication sets.
321
        # Symbolic ids for replication sets.
4585
285
        define lpmain_set  1;
322
        define lpmain_set   %d;
4586
286
        define authdb_set  2;
323
        define holding_set  %d;
4587
287
        define holding_set 666;
324
        define lpmirror_set %d;
4588
288
        """)]
325
        """ % (LPMAIN_SET_ID, HOLDING_SET_ID, LPMIRROR_SET_ID))]
4589
289
326
4590
290
    if master_node is not None:
327
    if master_node is not None:
4591
291
        preamble.append(dedent("""\
328
        preamble.append(dedent("""\
4592
@@ -303,9 +340,9 @@
4593
303
                node.nickname, node.node_id,
340
                node.nickname, node.node_id,
4594
304
                node.nickname, node.connection_string,
341
                node.nickname, node.connection_string,
4595
305
                node.nickname, node.nickname)))
342
                node.nickname, node.nickname)))
4597
306
    
343
4598
307
    return '\n\n'.join(preamble)
344
    return '\n\n'.join(preamble)
4600
308
        
345
4601
309
346
4602
310
def calculate_replication_set(cur, seeds):
347
def calculate_replication_set(cur, seeds):
4603
311
    """Return the minimal set of tables and sequences needed in a
348
    """Return the minimal set of tables and sequences needed in a
4604
@@ -434,7 +471,7 @@
4605
434
471
4606
435
    return (
472
    return (
4607
436
        all_tables - replicated_tables - IGNORED_TABLES,
473
        all_tables - replicated_tables - IGNORED_TABLES,
4609
437
        all_sequences - replicated_sequences)
474
        all_sequences - replicated_sequences - IGNORED_SEQUENCES)
4610
438
475
4611
439
476
4612
440
class ReplicationConfigError(Exception):
477
class ReplicationConfigError(Exception):
4613
@@ -462,19 +499,6 @@
4614
462
        raise ReplicationConfigError(
499
        raise ReplicationConfigError(
4615
463
            "Unreplicated sequences: %s" % repr(unrepl_sequences))
500
            "Unreplicated sequences: %s" % repr(unrepl_sequences))
4616
464
501
4617
465
    authdb_tables, authdb_sequences = calculate_replication_set(
4618
466
        cur, AUTHDB_SEED)
4619
467
    lpmain_tables, lpmain_sequences = calculate_replication_set(
502
    lpmain_tables, lpmain_sequences = calculate_replication_set(
4620
468
        cur, LPMAIN_SEED)
503
        cur, LPMAIN_SEED)
4621
469
504
4622
470
    confused_tables = authdb_tables.intersection(lpmain_tables)
4623
471
    if confused_tables:
4624
472
        raise ReplicationConfigError(
4625
473
            "Tables exist in multiple replication sets: %s"
4626
474
            % repr(confused_tables))
4627
475
    confused_sequences = authdb_sequences.intersection(lpmain_sequences)
4628
476
    if confused_sequences:
4629
477
        raise ReplicationConfigError(
4630
478
            "Sequences exist in multiple replication sets: %s"
4631
479
            % repr(confused_sequences))
4632
480
4633
481
505
4634
=== modified file 'database/replication/initialize.py'
4635
--- database/replication/initialize.py	2009-07-19 04:41:14 +0000
4636
+++ database/replication/initialize.py	2010-05-05 01:58:48 +0000
4637
@@ -1,4 +1,4 @@
4639
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
4640
2
#
2
#
4641
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4642
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4643
@@ -41,9 +41,9 @@
4644
41
    """Duplicate the master schema into the slaves."""
41
    """Duplicate the master schema into the slaves."""
4645
42
    log.info('Duplicating database schema')
42
    log.info('Duplicating database schema')
4646
43
43
4648
44
    master_cs = ConnectionString(config.database.main_master)
44
    master_cs = ConnectionString(config.database.rw_main_master)
4649
45
    master_cs.user = options.dbuser
45
    master_cs.user = options.dbuser
4651
46
    slave1_cs = ConnectionString(config.database.main_slave)
46
    slave1_cs = ConnectionString(config.database.rw_main_slave)
4652
47
    slave1_cs.user = options.dbuser
47
    slave1_cs.user = options.dbuser
4653
48
48
4654
49
    # We can't use pg_dump to replicate security as not all of the roles
49
    # We can't use pg_dump to replicate security as not all of the roles
4655
@@ -70,7 +70,7 @@
4656
70
    """Initialize the cluster."""
70
    """Initialize the cluster."""
4657
71
    log.info('Initializing Slony-I cluster')
71
    log.info('Initializing Slony-I cluster')
4658
72
    master_connection_string = ConnectionString(
72
    master_connection_string = ConnectionString(
4660
73
        config.database.main_master)
73
        config.database.rw_main_master)
4661
74
    master_connection_string.user = 'slony'
74
    master_connection_string.user = 'slony'
4662
75
    helpers.execute_slonik("""
75
    helpers.execute_slonik("""
4663
76
        node 1 admin conninfo = '%s';
76
        node 1 admin conninfo = '%s';
4664
@@ -88,54 +88,13 @@
4665
88
    helpers.sync(120) # Will exit on failure.
88
    helpers.sync(120) # Will exit on failure.
4666
89
89
4667
90
90
4670
91
def create_replication_sets(
91
def create_replication_sets(lpmain_tables, lpmain_sequences):
4669
92
    authdb_tables, authdb_sequences, lpmain_tables, lpmain_sequences):
4671
93
    """Create the replication sets."""
92
    """Create the replication sets."""
4672
94
    log.info('Creating Slony-I replication sets.')
93
    log.info('Creating Slony-I replication sets.')
4673
95
94
4674
96
    # Instead of creating both the authdb and lpmain replication sets,
4675
97
    # we just create the lpmain replication set containing everything.
4676
98
    # This way, we can then test the populate_auth_replication_set.py
4677
99
    # migration script that moves the relevant tables from the lpmain
4678
100
    # replication set to the authdb replication set.
4679
101
    # We will turn this behavior off once we are running two
4680
102
    # replication sets in production and remove the migration script.
4681
103
    lpmain_tables = lpmain_tables.union(authdb_tables)
4682
104
    lpmain_sequences = lpmain_sequences.union(authdb_sequences)
4683
105
4684
106
    script = ["try {"]
95
    script = ["try {"]
4685
107
    # script,append("""
4686
108
    #     echo 'Creating AuthDB replication set (@authdb_set)';
4687
109
    #     create set (
4688
110
    #         id=@authdb_set, origin=@master_node,
4689
111
    #         comment='AuthDB tables and sequences');
4690
112
    #     """)
4691
113
96
4717
114
    # entry_id = 1
97
    entry_id = 1
4693
115
    # for table in sorted(authdb_tables):
4694
116
    #     script.append("""
4695
117
    #         echo 'Adding %(table)s to replication set @authdb_set';
4696
118
    #         set add table (
4697
119
    #             set id=@authdb_set,
4698
120
    #             origin=@master_node,
4699
121
    #             id=%(entry_id)d,
4700
122
    #             fully qualified name='%(table)s');
4701
123
    #         """ % vars())
4702
124
    #     entry_id += 1
4703
125
    # entry_id = 1
4704
126
    # for sequence in sorted(authdb_sequences):
4705
127
    #     script.append("""
4706
128
    #         echo 'Adding %(sequence)s to replication set @authdb_set';
4707
129
    #         set add sequence (
4708
130
    #             set id=@authdb_set,
4709
131
    #             origin=@master_node,
4710
132
    #             id=%(entry_id)d,
4711
133
    #             fully qualified name='%(sequence)s');
4712
134
    #         """ % vars())
4713
135
    #     entry_id += 1
4714
136
    #
4715
137
    # assert entry_id < 200, 'authdb replcation set has > 200 objects???'
4716
138
    entry_id = 200
4718
139
98
4719
140
    script.append("""
99
    script.append("""
4720
141
        echo 'Creating LPMain replication set (@lpmain_set)';
100
        echo 'Creating LPMain replication set (@lpmain_set)';
4721
@@ -157,7 +116,7 @@
4722
157
            """ % vars())
116
            """ % vars())
4723
158
        entry_id += 1
117
        entry_id += 1
4724
159
118
4726
160
    entry_id = 200
119
    entry_id = 1
4727
161
    script.append(
120
    script.append(
4728
162
        "echo 'Adding %d sequences to replication set @lpmain_set';"
121
        "echo 'Adding %d sequences to replication set @lpmain_set';"
4729
163
        % len(lpmain_sequences))
122
        % len(lpmain_sequences))
4730
@@ -199,9 +158,6 @@
4731
199
    con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
158
    con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
4732
200
    global cur
159
    global cur
4733
201
    cur = con.cursor()
160
    cur = con.cursor()
4734
202
    log.debug("Calculating authdb replication set.")
4735
203
    authdb_tables, authdb_sequences = helpers.calculate_replication_set(
4736
204
        cur, helpers.AUTHDB_SEED)
4737
205
    log.debug("Calculating lpmain replication set.")
161
    log.debug("Calculating lpmain replication set.")
4738
206
    lpmain_tables, lpmain_sequences = helpers.calculate_replication_set(
162
    lpmain_tables, lpmain_sequences = helpers.calculate_replication_set(
4739
207
        cur, helpers.LPMAIN_SEED)
163
        cur, helpers.LPMAIN_SEED)
4740
@@ -212,8 +168,7 @@
4741
212
    fails = 0
168
    fails = 0
4742
213
    for table in all_tables_in_schema(cur, 'public'):
169
    for table in all_tables_in_schema(cur, 'public'):
4743
214
        times_seen = 0
170
        times_seen = 0
4746
215
        for table_set in [
171
        for table_set in [lpmain_tables, helpers.IGNORED_TABLES]:
4745
216
            authdb_tables, lpmain_tables, helpers.IGNORED_TABLES]:
4747
217
            if table in table_set:
172
            if table in table_set:
4748
218
                times_seen += 1
173
                times_seen += 1
4749
219
        if times_seen == 0:
174
        if times_seen == 0:
4750
@@ -224,7 +179,7 @@
4751
224
            fails += 1
179
            fails += 1
4752
225
    for sequence in all_sequences_in_schema(cur, 'public'):
180
    for sequence in all_sequences_in_schema(cur, 'public'):
4753
226
        times_seen = 0
181
        times_seen = 0
4755
227
        for sequence_set in [authdb_sequences, lpmain_sequences]:
182
        for sequence_set in [lpmain_sequences, helpers.IGNORED_SEQUENCES]:
4756
228
            if sequence in sequence_set:
183
            if sequence in sequence_set:
4757
229
                times_seen += 1
184
                times_seen += 1
4758
230
        if times_seen == 0:
185
        if times_seen == 0:
4759
@@ -241,8 +196,7 @@
4760
241
196
4761
242
    ensure_live()
197
    ensure_live()
4762
243
198
4765
244
    create_replication_sets(
199
    create_replication_sets(lpmain_tables, lpmain_sequences)
4764
245
        authdb_tables, authdb_sequences, lpmain_tables, lpmain_sequences)
4766
246
200
4767
247
    helpers.sync(0)
201
    helpers.sync(0)
4768
248
202
4769
249
203
4770
=== modified file 'database/replication/new-slave.py'
4771
--- database/replication/new-slave.py	2009-06-24 21:17:33 +0000
4772
+++ database/replication/new-slave.py	2010-05-05 01:58:48 +0000
4773
@@ -1,4 +1,4 @@
4775
1
#!/usr/bin/python2.4
1
#!/usr/bin/python2.5 -S
4776
2
#
2
#
4777
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4778
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4779
@@ -24,6 +24,7 @@
4780
24
from canonical.launchpad.scripts import db_options, logger_options, logger
24
from canonical.launchpad.scripts import db_options, logger_options, logger
4781
25
25
4782
26
import replication.helpers
26
import replication.helpers
4783
27
from replication.helpers import LPMAIN_SET_ID
4784
27
28
4785
28
def main():
29
def main():
4786
29
    parser = OptionParser(
30
    parser = OptionParser(
4787
@@ -72,6 +73,10 @@
4788
72
    if node_id in [node.node_id for node in existing_nodes]:
73
    if node_id in [node.node_id for node in existing_nodes]:
4789
73
        parser.error("Node %d already exists in the cluster." % node_id)
74
        parser.error("Node %d already exists in the cluster." % node_id)
4790
74
75
4791
76
    # Get the connection string for masters.
4792
77
    lpmain_connection_string = get_master_connection_string(
4793
78
        source_connection, parser, LPMAIN_SET_ID) or source_connection_string
4794
79
4795
75
    # Sanity check the target connection string.
80
    # Sanity check the target connection string.
4796
76
    target_connection_string = ConnectionString(raw_target_connection_string)
81
    target_connection_string = ConnectionString(raw_target_connection_string)
4797
77
    if target_connection_string.user is None:
82
    if target_connection_string.user is None:
4798
@@ -110,11 +115,11 @@
4799
110
            "Database at %s is not empty." % target_connection_string)
115
            "Database at %s is not empty." % target_connection_string)
4800
111
    target_con.rollback()
116
    target_con.rollback()
4801
112
117
4803
113
    # Duplicate the schema. We restore with no-privileges as required
118
    # Duplicate the full schema. We restore with no-privileges as required
4804
114
    # roles may not yet exist, so we have to run security.py on the
119
    # roles may not yet exist, so we have to run security.py on the
4805
115
    # new slave once it is built.
120
    # new slave once it is built.
4808
116
    log.info("Duplicating db schema from '%s' to '%s'" % (
121
    log.info("Duplicating full db schema from '%s' to '%s'" % (
4809
117
        source_connection_string, target_connection_string))
122
        lpmain_connection_string, target_connection_string))
4810
118
    cmd = "pg_dump --schema-only --no-privileges %s | psql -1 -q %s" % (
123
    cmd = "pg_dump --schema-only --no-privileges %s | psql -1 -q %s" % (
4811
119
        source_connection_string.asPGCommandLineArgs(),
124
        source_connection_string.asPGCommandLineArgs(),
4812
120
        target_connection_string.asPGCommandLineArgs())
125
        target_connection_string.asPGCommandLineArgs())
4813
@@ -123,19 +128,39 @@
4814
123
        return 1
128
        return 1
4815
124
129
4816
125
    # Trash the broken Slony tables we just duplicated.
130
    # Trash the broken Slony tables we just duplicated.
4817
131
    log.debug("Removing slony cruft.")
4818
126
    cur = target_con.cursor()
132
    cur = target_con.cursor()
4819
127
    cur.execute("DROP SCHEMA _sl CASCADE")
133
    cur.execute("DROP SCHEMA _sl CASCADE")
4820
128
    target_con.commit()
134
    target_con.commit()
4821
129
    del target_con
135
    del target_con
4822
130
136
4824
131
    # Get a list of existing set ids.
137
    # Get a list of existing set ids that can be subscribed too. This
4825
138
    # is all sets where the origin is the master_node. We
4826
139
    # don't allow other sets where the master is configured as a
4827
140
    # forwarding slave as we have to special case rebuilding the database
4828
141
    # schema, and we want to avoid cascading slave configurations anyway
4829
142
    # since we are running an antique Slony-I at the moment - keep it
4830
143
    # simple!
4831
144
    # We order the sets smallest to largest by number of tables.
4832
145
    # This should let us subscribe the quickest sets first for more
4833
146
    # immediate feedback.
4834
132
    source_connection.rollback()
147
    source_connection.rollback()
4835
133
    master_node = replication.helpers.get_master_node(source_connection)
148
    master_node = replication.helpers.get_master_node(source_connection)
4836
134
    cur = source_connection.cursor()
149
    cur = source_connection.cursor()
4840
135
    cur.execute(
150
    cur.execute("""
4841
136
        "SELECT set_id FROM _sl.sl_set WHERE set_origin=%d"
151
        SELECT set_id
4842
137
        % master_node.node_id)
152
        FROM _sl.sl_set, (
4843
153
            SELECT tab_set, count(*) AS tab_count
4844
154
            FROM _sl.sl_table GROUP BY tab_set
4845
155
            ) AS TableCounts
4846
156
        WHERE
4847
157
            set_origin=%d
4848
158
            AND tab_set = set_id
4849
159
        ORDER BY tab_count
4850
160
        """
4851
161
        % (master_node.node_id,))
4852
138
    set_ids = [set_id for set_id, in cur.fetchall()]
162
    set_ids = [set_id for set_id, in cur.fetchall()]
4853
163
    log.debug("Discovered set ids %s" % repr(list(set_ids)))
4854
139
164
4855
140
    # Generate and run a slonik(1) script to initialize the new node
165
    # Generate and run a slonik(1) script to initialize the new node
4856
141
    # and subscribe it to our replication sets.
166
    # and subscribe it to our replication sets.
4857
@@ -165,19 +190,32 @@
4858
165
        } on error { echo 'Failed.'; exit 1; }
190
        } on error { echo 'Failed.'; exit 1; }
4859
166
        """)
191
        """)
4860
167
192
4861
193
    full_sync = []
4862
194
    sync_nicknames = [node.nickname for node in existing_nodes]
4863
195
    sync_nicknames.append('new_node');
4864
196
    for nickname in sync_nicknames:
4865
197
        full_sync.append(dedent("""\
4866
198
            echo 'Waiting for %(nickname)s sync.';
4867
199
            sync (id=@%(nickname)s);
4868
200
            wait for event (
4869
201
                origin = @%(nickname)s, confirmed=ALL,
4870
202
                wait on = @%(nickname)s, timeout=0);
4871
203
            """ % {'nickname': nickname}))
4872
204
    full_sync = '\n'.join(full_sync)
4873
205
    script += full_sync
4874
206
4875
168
    for set_id in set_ids:
207
    for set_id in set_ids:
4876
169
4877
170
        script += dedent("""\
208
        script += dedent("""\
4878
171
        echo 'Subscribing new node to set %d.';
209
        echo 'Subscribing new node to set %d.';
4879
172
        subscribe set (
210
        subscribe set (
4880
173
            id=%d, provider=@master_node, receiver=@new_node, forward=yes);
211
            id=%d, provider=@master_node, receiver=@new_node, forward=yes);
4883
174
212
        echo 'Waiting for subscribe to start processing.';
4882
175
        echo 'Waiting for sync... this might take a while...';
4884
176
        sync (id = @master_node);
213
        sync (id = @master_node);
4885
177
        wait for event (
214
        wait for event (
4887
178
            origin = ALL, confirmed = ALL,
215
            origin = @master_node, confirmed = ALL,
4888
179
            wait on = @master_node, timeout = 0);
216
            wait on = @master_node, timeout = 0);
4889
180
        """ % (set_id, set_id))
217
        """ % (set_id, set_id))
4890
218
        script += full_sync
4891
181
219
4892
182
    replication.helpers.execute_slonik(script)
220
    replication.helpers.execute_slonik(script)
4893
183
221
4894
@@ -185,5 +223,33 @@
4895
185
223
4896
186
    return 0
224
    return 0
4897
187
225
4898
226
4899
227
def get_master_connection_string(con, parser, set_id):
4900
228
    """Return the connection string to the origin for the replication set.
4901
229
    """
4902
230
    cur = con.cursor()
4903
231
    cur.execute("""
4904
232
        SELECT pa_conninfo FROM _sl.sl_set, _sl.sl_path
4905
233
        WHERE set_origin = pa_server AND set_id = %d
4906
234
        LIMIT 1
4907
235
        """ % set_id)
4908
236
    row = cur.fetchone()
4909
237
    if row is None:
4910
238
        # If we have no paths stored, there is only a single node in the
4911
239
        # cluster.
4912
240
        return None
4913
241
    else:
4914
242
        connection_string = ConnectionString(row[0])
4915
243
4916
244
    # Confirm we can connect from here.
4917
245
    try:
4918
246
        test_con = psycopg2.connect(str(connection_string))
4919
247
    except psycopg2.Error, exception:
4920
248
        parser.error("Failed to connect to using '%s' (%s)" % (
4921
249
            connection_string, str(exception).strip()))
4922
250
4923
251
    return connection_string
4924
252
4925
253
4926
188
if __name__ == '__main__':
254
if __name__ == '__main__':
4927
189
    sys.exit(main())
255
    sys.exit(main())
4928
190
256
4929
=== removed file 'database/replication/populate_auth_replication_set.py'
4930
--- database/replication/populate_auth_replication_set.py	2009-06-24 21:17:33 +0000
4931
+++ database/replication/populate_auth_replication_set.py	1970-01-01 00:00:00 +0000
4932
@@ -1,177 +0,0 @@
4933
1
#!/usr/bin/python2.4
4934
2
#
4935
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4936
4
# GNU Affero General Public License version 3 (see the file LICENSE).
4937
5
4938
6
"""Populate the auth replication set.
4939
7
4940
8
This script moves the the SSO tables from the main replication set to
4941
9
the auth replication set.
4942
10
4943
11
Once it has been run on production, these tables can no longer be
4944
12
maintained using the Launchpad database maintenance scripts
4945
13
(upgrade.py, security.py etc.).
4946
14
4947
15
We do this so Launchpad database upgrades do not lock the SSO tables,
4948
16
allowing the SSO service to continue to operate.
4949
17
4950
18
This is a single shot script.
4951
19
"""
4952
20
4953
21
__metaclass__ = type
4954
22
__all__ = []
4955
23
4956
24
import _pythonpath
4957
25
4958
26
import sys
4959
27
from textwrap import dedent
4960
28
from optparse import OptionParser
4961
29
4962
30
from canonical.database.sqlbase import (
4963
31
    connect, ISOLATION_LEVEL_AUTOCOMMIT, sqlvalues)
4964
32
from canonical.launchpad.scripts import db_options, logger_options, logger
4965
33
4966
34
import replication.helpers
4967
35
4968
36
def create_auth_set(cur):
4969
37
    """Create the auth replication set if it doesn't already exist."""
4970
38
    cur.execute("SELECT TRUE FROM _sl.sl_set WHERE set_id=2")
4971
39
    if cur.fetchone() is not None:
4972
40
        log.info("Auth set already exists.")
4973
41
        return
4974
42
    slonik_script = dedent("""\
4975
43
        create set (
4976
44
            id=@authdb_set, origin=@master_node,
4977
45
            comment='SSO service tables');
4978
46
        """)
4979
47
    log.info("Creating authdb replication set.")
4980
48
    replication.helpers.execute_slonik(slonik_script, sync=0)
4981
49
4982
50
4983
51
def subscribe_auth_set(cur):
4984
52
    """The authdb set subscription much match the lpmain set subscription.
4985
53
4986
54
    This is a requirement to move stuff between replication sets. It
4987
55
    is also what we want (all nodes replicating everything).
4988
56
    """
4989
57
    cur.execute("""
4990
58
        SELECT sub_receiver FROM _sl.sl_subscribe WHERE sub_set = 1
4991
59
        EXCEPT
4992
60
        SELECT sub_receiver FROM _sl.sl_subscribe WHERE sub_set = 2
4993
61
        """)
4994
62
    for node_id in (node_id for node_id, in cur.fetchall()):
4995
63
        log.info("Subscribing Node #%d to authdb replication set" % node_id)
4996
64
        success = replication.helpers.execute_slonik(dedent("""\
4997
65
            subscribe set (
4998
66
                id = @authdb_set, provider = @master_node,
4999
67
                receiver = %d, forward = yes);
5000
68
            """ % node_id), sync=0)
Reviewer	Review Type	Date Requested	Status
Francis J. Lacoste (community)	release-critical	2010-05-03	Approve on 2010-05-04
Curtis Hovey (community)			Approve on 2010-05-03
Brad Crittenden			Pending
Review via email: mp+24625@code.launchpad.net