1
=== modified file 'buildout-templates/bin/test.in'
2
--- buildout-templates/bin/test.in	2010-07-30 11:52:48 +0000
3
+++ buildout-templates/bin/test.in	2010-08-17 13:34:50 +0000
4
@@ -45,15 +45,6 @@
5
45
BUILD_DIR = ${buildout:directory|path-repr}
45
BUILD_DIR = ${buildout:directory|path-repr}
6
46
CUSTOM_SITE_DIR = ${scripts:parts-directory|path-repr}
46
CUSTOM_SITE_DIR = ${scripts:parts-directory|path-repr}
7
47
47
8
48
if os.getsid(0) == os.getsid(os.getppid()):
9
49
    # We need to become the process group leader so test_on_merge.py
10
50
    # can reap its children.
11
51
    #
12
52
    # Note that if setpgrp() is used to move a process from one
13
53
    # process group to another (as is done by some shells when
14
54
    # creating pipelines), then both process groups must be part of
15
55
    # the same session.
16
56
    os.setpgrp()
17
57
48
18
58
# Make tests run in a timezone no launchpad developers live in.
49
# Make tests run in a timezone no launchpad developers live in.
19
59
# Our tests need to run in any timezone.
50
# Our tests need to run in any timezone.
20
60
51
21
=== modified file 'test_on_merge.py'
22
--- test_on_merge.py	2010-07-28 14:28:57 +0000
23
+++ test_on_merge.py	2010-08-17 13:34:50 +0000
24
@@ -12,42 +12,39 @@
25
12
from StringIO import StringIO
12
from StringIO import StringIO
26
13
import psycopg2
13
import psycopg2
27
14
from subprocess import Popen, PIPE, STDOUT
14
from subprocess import Popen, PIPE, STDOUT
29
15
from signal import SIGKILL, SIGTERM
15
from signal import SIGKILL, SIGTERM, SIGINT, SIGHUP
30
16
from select import select
16
from select import select
31
17
17
32
18
33
18
# The TIMEOUT setting (expressed in seconds) affects how long a test will run
19
# The TIMEOUT setting (expressed in seconds) affects how long a test will run
34
19
# before it is deemed to be hung, and then appropriately terminated.
20
# before it is deemed to be hung, and then appropriately terminated.
35
20
# It's principal use is preventing a PQM job from hanging indefinitely and
21
# It's principal use is preventing a PQM job from hanging indefinitely and
36
21
# backing up the queue.
22
# backing up the queue.
40
22
# e.g. Usage: TIMEOUT = 60 * 15
23
# e.g. Usage: TIMEOUT = 60 * 10
41
23
# This will set the timeout to 15 minutes.
24
# This will set the timeout to 10 minutes.
42
24
TIMEOUT = 60 * 15
25
TIMEOUT = 60 * 10
43
26
44
27
HERE = os.path.dirname(os.path.realpath(__file__))
45
28
46
25
29
47
26
def main():
30
def main():
48
27
    """Call bin/test with whatever arguments this script was run with.
31
    """Call bin/test with whatever arguments this script was run with.
49
28
32
52
29
    If the tests ran ok (last line of stderr is 'OK<return>') then suppress
33
    Prior to running the tests this script sets up the test database.
51
30
    output and exit(0).
53
31
34
55
32
    Otherwise, print output and exit(1).
35
    Returns 1 on error, otherwise it returns the testrunner's exit code.
56
33
    """
36
    """
72
34
    here = os.path.dirname(os.path.realpath(__file__))
37
    if setup_test_database() != 0:
58
35
59
36
    # Tabnanny
60
37
    # NB. If tabnanny raises an exception, run
61
38
    # python /usr/lib/python2.5/tabnanny.py -vv lib/canonical
62
39
    # for more detailed output.
63
40
    org_stdout = sys.stdout
64
41
    sys.stdout = StringIO()
65
42
    tabnanny.check(os.path.join(here, 'lib', 'canonical'))
66
43
    tabnanny_results = sys.stdout.getvalue()
67
44
    sys.stdout = org_stdout
68
45
    if len(tabnanny_results) > 0:
69
46
        print '---- tabnanny bitching ----'
70
47
        print tabnanny_results
71
48
        print '---- end tabnanny bitching ----'
73
49
        return 1
38
        return 1
74
50
39
75
40
    return run_test_process()
76
41
77
42
78
43
def setup_test_database():
79
44
    """Set up a test instance of our postgresql database.
80
45
81
46
    Returns 0 for success, 1 for errors.
82
47
    """
83
51
    # Sanity check PostgreSQL version. No point in trying to create a test
48
    # Sanity check PostgreSQL version. No point in trying to create a test
84
52
    # database when PostgreSQL is too old.
49
    # database when PostgreSQL is too old.
85
53
    con = psycopg2.connect('dbname=template1')
50
    con = psycopg2.connect('dbname=template1')
86
@@ -91,8 +88,7 @@
87
91
    con.close()
88
    con.close()
88
92
89
89
93
    # Build the template database. Tests duplicate this.
90
    # Build the template database. Tests duplicate this.
92
94
    here = os.path.dirname(os.path.realpath(__file__))
91
    schema_dir = os.path.join(HERE, 'database', 'schema')
91
95
    schema_dir = os.path.join(here, 'database', 'schema')
93
96
    if os.system('cd %s; make test > /dev/null' % (schema_dir)) != 0:
92
    if os.system('cd %s; make test > /dev/null' % (schema_dir)) != 0:
94
97
        print 'Failed to create database or load sampledata.'
93
        print 'Failed to create database or load sampledata.'
95
98
        return 1
94
        return 1
96
@@ -134,70 +130,118 @@
97
134
    con.close()
130
    con.close()
98
135
    del con
131
    del con
99
136
132
100
133
    return 0
101
134
102
135
103
136
def run_test_process():
104
137
    """Start the testrunner process and return its exit code."""
105
137
    print 'Running tests.'
138
    print 'Running tests.'
107
138
    os.chdir(here)
139
    os.chdir(HERE)
108
140
109
141
    # We run the test suite under a virtual frame buffer server so that the
110
142
    # JavaScript integration test suite can run.
111
139
    cmd = [
143
    cmd = [
113
140
        'xvfb-run',
144
        '/usr/bin/xvfb-run',
114
141
        "--error-file=/var/tmp/xvfb-errors.log",
145
        "--error-file=/var/tmp/xvfb-errors.log",
115
142
        "--server-args='-screen 0 1024x768x24'",
146
        "--server-args='-screen 0 1024x768x24'",
117
143
        os.path.join(here, 'bin', 'test')] + sys.argv[1:]
147
        os.path.join(HERE, 'bin', 'test')] + sys.argv[1:]
118
144
    command_line = ' '.join(cmd)
148
    command_line = ' '.join(cmd)
131
145
    print command_line
149
    print "Running command:", command_line
132
146
150
133
147
    # Run the test suite and return the error code
151
    # Run the test suite.  Make the suite the leader of a new process group
134
148
    #return call(cmd)
152
    # so that we can signal the group without signaling ourselves.
135
149
153
    xvfb_proc = Popen(
136
150
    proc = Popen(
154
        command_line,
137
151
        command_line, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True)
155
        stdout=PIPE,
138
152
    proc.stdin.close()
156
        stderr=STDOUT,
139
153
157
        preexec_fn=os.setpgrp,
140
154
    # Do proc.communicate(), but timeout if there's no activity on stdout or
158
        shell=True)
141
155
    # stderr for too long.
159
142
156
    open_readers = set([proc.stdout])
160
    # This code is very similar to what takes place in Popen._communicate(),
143
161
    # but this code times out if there is no activity on STDOUT for too long.
144
162
    # This keeps us from blocking when reading from a hung testrunner, allows
145
163
    # us to time out if the child process hangs, and avoids issues when using
146
164
    # Popen.communicate() with large data sets.
147
165
    open_readers = set([xvfb_proc.stdout])
148
157
    while open_readers:
166
    while open_readers:
149
158
        rlist, wlist, xlist = select(open_readers, [], [], TIMEOUT)
167
        rlist, wlist, xlist = select(open_readers, [], [], TIMEOUT)
150
159
168
151
160
        if len(rlist) == 0:
169
        if len(rlist) == 0:
153
161
            if proc.poll() is not None:
170
            # The select() statement timed out!
154
171
155
172
            if xvfb_proc.poll() is not None:
156
173
                # The process we were watching died.
157
162
                break
174
                break
167
163
            print ("\nA test appears to be hung. There has been no output for"
175
168
164
                " %d seconds. Sending SIGTERM." % TIMEOUT)
176
            cleanup_hung_testrunner(xvfb_proc)
160
165
            killem(proc.pid, SIGTERM)
161
166
            time.sleep(3)
162
167
            if proc.poll() is not None:
163
168
                print ("\nSIGTERM did not work. Sending SIGKILL.")
164
169
                killem(proc.pid, SIGKILL)
165
170
            # Drain the subprocess's stdout and stderr.
166
171
            sys.stdout.write(proc.stdout.read())
169
172
            break
177
            break
170
173
178
173
174
        if proc.stdout in rlist:
179
        if xvfb_proc.stdout in rlist:
174
175
            chunk = os.read(proc.stdout.fileno(), 1024)
180
            # Read a chunk of output from STDOUT.
175
181
            chunk = os.read(xvfb_proc.stdout.fileno(), 1024)
176
176
            sys.stdout.write(chunk)
182
            sys.stdout.write(chunk)
177
177
            if chunk == "":
183
            if chunk == "":
181
178
                open_readers.remove(proc.stdout)
184
                # Gracefully exit the loop if STDOUT is empty.
182
179
185
                open_readers.remove(xvfb_proc.stdout)
183
180
    rv = proc.wait()
186
184
187
    rv = xvfb_proc.wait()
185
188
186
181
    if rv == 0:
189
    if rv == 0:
188
182
        print '\nSuccessfully ran all tests.'
190
        print
189
191
        print 'Successfully ran all tests.'
190
183
    else:
192
    else:
192
184
        print '\nTests failed (exit code %d)' % rv
193
        print
193
194
        print 'Tests failed (exit code %d)' % rv
194
185
195
195
186
    return rv
196
    return rv
196
187
197
197
188
198
203
189
def killem(pid, signal):
199
def cleanup_hung_testrunner(process):
204
190
    """Kill the process group leader identified by pid and other group members
200
    """Kill and clean up the testrunner process and its children."""
205
191
201
    print
206
192
    Note that bin/test sets its process to a process group leader.
202
    print
207
193
    """
203
    print ("WARNING: A test appears to be hung. There has been no "
208
204
        "output for %d seconds." % TIMEOUT)
209
205
    print "Forcibly shutting down the test suite"
210
206
211
207
    # This guarantees the process will die.  In rare cases
212
208
    # a child process may survive this if they are in a different
213
209
    # process group and they ignore the signals we send their parent.
214
210
    nice_killpg(process.pid)
215
211
216
212
    # The process should absolutely be dead now.
217
213
    assert process.poll() is not None
218
214
219
215
    # Drain the subprocess's stdout and stderr.
220
216
    print "The dying processes left behind the following output:"
221
217
    print "--------------- BEGIN OUTPUT ---------------"
222
218
    sys.stdout.write(process.stdout.read())
223
219
    print
224
220
    print "---------------- END OUTPUT ----------------"
225
221
226
222
227
223
def nice_killpg(pgid):
228
224
    """Kill a Unix process group using increasingly harmful signals."""
229
194
    try:
225
    try:
233
195
        os.killpg(os.getpgid(pid), signal)
226
        print "Process group %d will be killed" % pgid
234
196
    except OSError, x:
227
235
197
        if x.errno == errno.ESRCH:
228
        # Attempt a series of increasingly brutal methods of killing the
236
229
        # process.
237
230
        for signum in [SIGTERM, SIGINT, SIGHUP, SIGKILL]:
238
231
            print "Sending signal %s to process group %d" % (signum, pgid)
239
232
            os.killpg(pgid, signum)
240
233
241
234
            # Give the processes some time to shut down.
242
235
            time.sleep(3)
243
236
244
237
    except OSError, exc:
245
238
        if exc.errno == errno.ESRCH:
246
239
            # We tried to call os.killpg() and found the group to be empty.
247
198
            pass
240
            pass
248
199
        else:
241
        else:
249
200
            raise
242
            raise
250
243
    print "Process group %d is now empty." % pgid
251
244
252
201
245
253
202
if __name__ == '__main__':
246
if __name__ == '__main__':
254
203
    sys.exit(main())
247
    sys.exit(main())
Reviewer	Date Requested	Status
Michael Hudson-Doyle	2010-07-28	Approve on 2010-08-04
Robert Collins	2010-07-28	Pending
Māris Fogels		Pending
Review via email: mp+31207@code.launchpad.net