1
=== modified file 'lib/canonical/testing/layers.py'
2
--- lib/canonical/testing/layers.py	2010-02-12 19:34:42 +0000
3
+++ lib/canonical/testing/layers.py	2010-03-06 08:12:37 +0000
4
@@ -537,6 +537,11 @@
5
537
    def getPidFile(cls):
537
    def getPidFile(cls):
6
538
        return os.path.join(config.root, '.memcache.pid')
538
        return os.path.join(config.root, '.memcache.pid')
7
539
539
8
540
    @classmethod
9
541
    def purge(cls):
10
542
        "Purge everything from our memcached."
11
543
        MemcachedLayer.client.flush_all() # Only do this in tests!
12
544
13
540
545
14
541
class LibrarianLayer(BaseLayer):
546
class LibrarianLayer(BaseLayer):
15
542
    """Provides tests access to a Librarian instance.
547
    """Provides tests access to a Librarian instance.
16
543
548
17
=== modified file 'lib/lp/app/stories/launchpad-root/xx-featuredprojects.txt'
18
--- lib/lp/app/stories/launchpad-root/xx-featuredprojects.txt	2010-01-08 21:23:15 +0000
19
+++ lib/lp/app/stories/launchpad-root/xx-featuredprojects.txt	2010-03-06 08:12:37 +0000
20
@@ -78,6 +78,9 @@
21
78
Administrators can add a project. Here Foo Bar adds apache as a featured
78
Administrators can add a project. Here Foo Bar adds apache as a featured
22
79
project:
79
project:
23
80
80
24
81
    >>> from canonical.testing.layers import MemcachedLayer
25
82
    >>> MemcachedLayer.purge() # Featured projects list is cached.
26
83
27
81
    >>> admin_browser.getControl('Add project').value = 'apache'
84
    >>> admin_browser.getControl('Add project').value = 'apache'
28
82
    >>> admin_browser.getControl('Update').click()
85
    >>> admin_browser.getControl('Update').click()
29
83
    >>> admin_browser.url
86
    >>> admin_browser.url
30
@@ -111,6 +114,8 @@
31
111
114
32
112
== Removing a project ==
115
== Removing a project ==
33
113
116
34
117
    >>> MemcachedLayer.purge() # Featured projects list is cached.
35
118
36
114
    >>> admin_browser.getLink(MANAGE_LINK).click()
119
    >>> admin_browser.getLink(MANAGE_LINK).click()
37
115
    >>> admin_browser.getControl('Apache').click()
120
    >>> admin_browser.getControl('Apache').click()
38
116
    >>> admin_browser.getControl('Update').click()
121
    >>> admin_browser.getControl('Update').click()
39
117
122
40
=== modified file 'lib/lp/app/templates/root-index.pt'
41
--- lib/lp/app/templates/root-index.pt	2010-02-22 17:58:40 +0000
42
+++ lib/lp/app/templates/root-index.pt	2010-03-06 08:12:37 +0000
43
@@ -86,7 +86,7 @@
44
86
        <div class="yui-g">
86
        <div class="yui-g">
45
87
          <div class="yui-u first">
87
          <div class="yui-u first">
46
88
            <div class="homepage-whatslaunchpad"
88
            <div class="homepage-whatslaunchpad"
48
89
                 tal:condition="not:view/user">
89
                 tal:condition="not:view/user" tal:content="cache:anonymous">
49
90
              <h2><span class="launchpad-gold">Launchpad</span> is a software collaboration platform that provides:</h2>
90
              <h2><span class="launchpad-gold">Launchpad</span> is a software collaboration platform that provides:</h2>
50
91
              <ul tal:define="apphomes view/apphomes">
91
              <ul tal:define="apphomes view/apphomes">
51
92
              <li><a tal:attributes="href apphomes/bugs"><img src="/@@/bug" alt="" /></a>
92
              <li><a tal:attributes="href apphomes/bugs"><img src="/@@/bug" alt="" /></a>
52
@@ -159,7 +159,7 @@
53
159
              <input id="text" type="text" name="field.text" size="25%" />
159
              <input id="text" type="text" name="field.text" size="25%" />
54
160
              <input id="search" type="submit" value="Search Launchpad" />
160
              <input id="search" type="submit" value="Search Launchpad" />
55
161
            </form>
161
            </form>
57
162
            <div id="homepage-stats">
162
            <div id="homepage-stats" tal:content="cache:public, 1 hour">
58
163
              <strong class="registry-stat"
163
              <strong class="registry-stat"
59
164
                tal:content="view/project_count/fmt:intcomma">123</strong>&nbsp;projects,
164
                tal:content="view/project_count/fmt:intcomma">123</strong>&nbsp;projects,
60
165
              <strong class="bugs-stat"
165
              <strong class="bugs-stat"
61
@@ -182,7 +182,8 @@
62
182
              </tal:logged_out>You can test Launchpad's functionality
182
              </tal:logged_out>You can test Launchpad's functionality
63
183
              in our sandbox environment.
183
              in our sandbox environment.
64
184
              (<a href="/+help/home-page-staging-help.html" target="help">What's this?</a>)<br />
184
              (<a href="/+help/home-page-staging-help.html" target="help">What's this?</a>)<br />
66
185
              <tal:logged_in condition="view/user" omit-tag="">
185
              <tal:logged_in condition="view/user" omit-tag=""
67
186
                  tal:content="cache:public">
68
186
                If you're ready, you can:
187
                If you're ready, you can:
69
187
                <ul tal:define="apphomes view/apphomes">
188
                <ul tal:define="apphomes view/apphomes">
70
188
                  <li><a href="https://help.launchpad.net/">
189
                  <li><a href="https://help.launchpad.net/">
71
@@ -207,7 +208,10 @@
72
207
              </tal:logged_in>
208
              </tal:logged_in>
73
208
            </div>
209
            </div>
74
209
210
76
210
            <div id="homepage-featured" class="homepage-portlet">
211
            <div id="homepage-featured" class="homepage-portlet"
77
212
                tal:content="cache:anonymous, 1 hour">
78
213
              <tal:cache
79
214
                  tal:content="cache:public, 5 minutes" tal:omit-tag="">
80
211
              <h2>Featured projects</h2>
215
              <h2>Featured projects</h2>
81
212
216
82
213
              <div class="featured-project-top"
217
              <div class="featured-project-top"
83
@@ -231,9 +235,10 @@
84
231
                  </li>
235
                  </li>
85
232
                </ul>
236
                </ul>
86
233
              </div>
237
              </div>
87
238
              </tal:cache>
88
234
239
89
235
              <ul class="horizontal">
240
              <ul class="horizontal">
91
236
                <li>
241
                <li tal:content="cache:public, 1 hour">
92
237
                  <strong><a href="/projects">Browse all
242
                  <strong><a href="/projects">Browse all
93
238
                    <tal:count content="view/project_count">42</tal:count>
243
                    <tal:count content="view/project_count">42</tal:count>
94
239
                    projects</a>!</strong>
244
                    projects</a>!</strong>
95
240
245
96
=== modified file 'lib/lp/services/memcache/configure.zcml'
97
--- lib/lp/services/memcache/configure.zcml	2009-09-16 12:47:23 +0000
98
+++ lib/lp/services/memcache/configure.zcml	2010-03-06 08:12:37 +0000
99
@@ -5,10 +5,16 @@
100
5
    xmlns="http://namespaces.zope.org/zope"
5
    xmlns="http://namespaces.zope.org/zope"
101
6
    xmlns:browser="http://namespaces.zope.org/browser"
6
    xmlns:browser="http://namespaces.zope.org/browser"
102
7
    xmlns:i18n="http://namespaces.zope.org/i18n"
7
    xmlns:i18n="http://namespaces.zope.org/i18n"
103
8
    xmlns:tales="http://namespaces.zope.org/tales"
104
8
    i18n_domain="launchpad">
9
    i18n_domain="launchpad">
105
10
106
11
    <!-- Main memcache interface - the IMemcacheClient Utility -->
107
9
    <utility
12
    <utility
108
10
        provides="lp.services.memcache.interfaces.IMemcacheClient"
13
        provides="lp.services.memcache.interfaces.IMemcacheClient"
109
11
        factory="lp.services.memcache.client.memcache_client_factory"
14
        factory="lp.services.memcache.client.memcache_client_factory"
110
12
        />
15
        />
111
16
112
17
    <!-- TALES expression letting us cache chunks of rendered templates -->
113
18
    <tales:expressiontype name="cache" handler=".tales.MemcacheExpr" />
114
13
</configure>
19
</configure>
115
14
20
116
15
21
117
=== added directory 'lib/lp/services/memcache/doc'
118
=== added file 'lib/lp/services/memcache/doc/tales-cache.txt'
119
--- lib/lp/services/memcache/doc/tales-cache.txt	1970-01-01 00:00:00 +0000
120
+++ lib/lp/services/memcache/doc/tales-cache.txt	2010-03-06 08:12:37 +0000
121
@@ -0,0 +1,214 @@
122
1
Memcache with TALES
123
2
===================
124
3
125
4
We have extended TALES with a cache: expression to allow chunks of
126
5
rendered page templates to be cached in Memcached.
127
6
128
7
129
8
    >>> template = TestPageTemplate(dedent("""\
130
9
    ...     <div tal:content="cache:public">
131
10
    ...         <span tal:content="param">placeholder</span>
132
11
    ...     </div>"""))
133
12
134
13
135
14
The first time we render the page template, there is no information
136
15
in the cache. The cachable section is interpreted and stored in the cache
137
16
for next time.
138
17
139
18
    >>> print template(param='first')
140
19
    <div>
141
20
        <span>first</span>
142
21
    </div>
143
22
144
23
145
24
The second time we render the page template, the cached information
146
25
is used. We prove this here by changing our parameters, which would
147
26
cause this template to render differently.
148
27
149
28
    >>> print template(param='second')
150
29
    <div>
151
30
        <span>first</span>
152
31
    </div>
153
32
154
33
155
34
If we clear the cache, it will be rendered as expected.
156
35
157
36
    >>> MemcachedLayer.purge()
158
37
    >>> print template(param='third')
159
38
    <div>
160
39
        <span>third</span>
161
40
    </div>
162
41
163
42
164
43
Expiry
165
44
------
166
45
167
46
We can specify how long cached information is considered valid. If
168
47
this is not set, the information may be cached indefinitely. Note
169
48
that memcache may evict information sooner if it runs low on storage
170
49
space.
171
50
172
51
One interesting technique is to specify a lengthy expiry, but to
173
52
refresh the information asynchronously using an AJAX request. This
174
53
is good enough for bots and improves the initial page load time, but
175
54
care will be needed to avoid 'popping'.
176
55
177
56
    >>> template = TestPageTemplate(dedent("""\
178
57
    ...     <body tal:omit-tag="">
179
58
    ...         <div tal:content="cache:public,30 seconds" tal:omit-tag="">
180
59
    ...             This bit cached up to 30 seconds.
181
60
    ...         </div>
182
61
    ...         <div tal:content="cache:public,1 minute" tal:omit-tag="">
183
62
    ...             This bit cached up to 1 minute.
184
63
    ...         </div>
185
64
    ...         <div tal:content="cache:public,6 hours" tal:omit-tag="">
186
65
    ...             This bit cached up to 6 hours.
187
66
    ...         </div>
188
67
    ...         <tal:cached content="cache:public,3 days">
189
68
    ...             This bit cached up to 3 days.
190
69
    ...         </tal:cached>
191
70
    ...     </body>"""))
192
71
    >>> print template()
193
72
    This bit cached up to 30 seconds.
194
73
    This bit cached up to 1 minute.
195
74
    This bit cached up to 6 hours.
196
75
    This bit cached up to 3 days.
197
76
198
77
199
78
Visibility
200
79
----------
201
80
202
81
We define 4 different types of 'visibility':
203
82
204
83
    public
205
84
206
85
        The cached information is shared by everyone. These sections
207
86
        should not be personalized. They can contain private information
208
87
        if the page itself is protected.
209
88
210
89
    private
211
90
212
91
        Unauthenticated users share cached information, but
213
92
        authenticated users do not share with anyone else. A list on a
214
93
        publicly accessible page that might contain private information
215
94
        should use this visibility.
216
95
217
96
    anonymous
218
97
219
98
        Unauthenticated users share cached information, but
220
99
        authenticated users do not use the cache at all. This can
221
100
        be used to feed bots cached information quicky, while giving
222
101
        authenticated users up to date information. In practice, this
223
102
        might not make much difference as reverse proxies should
224
103
        already be caching the entire page for unauthenticated users.
225
104
226
105
    authenticated
227
106
228
107
        Unauthenticated users share cached information, and all
229
108
        authenticated users share a different cache. This is used
230
109
        when information is being hidden from unauthenticated users,
231
110
        for example when we hide email addresses from unauthenticated
232
111
        users to help protect against email address harvesters.
233
112
234
113
    >>> template = TestPageTemplate(dedent("""\
235
114
    ...     <div tal:omit-tag="">
236
115
    ...         <tal:cache content="cache:public">
237
116
    ...             Public: <tal:x content="username" />
238
117
    ...         </tal:cache>
239
118
    ...         <tal:cache content="cache:private">
240
119
    ...             Private: <tal:x content="username" />
241
120
    ...         </tal:cache>
242
121
    ...         <tal:cache content="cache:anonymous">
243
122
    ...             Anonymous: <tal:x content="username" />
244
123
    ...         </tal:cache>
245
124
    ...         <tal:cache content="cache:authenticated">
246
125
    ...             Authenticated: <tal:x content="username" />
247
126
    ...         </tal:cache>
248
127
    ...     </div>"""))
249
128
250
129
Here we populate all caches.
251
130
252
131
    >>> login(ANONYMOUS)
253
132
    >>> print template(username="Anonymous")
254
133
    Public:        Anonymous
255
134
    Private:       Anonymous
256
135
    Anonymous:     Anonymous
257
136
    Authenticated: Anonymous
258
137
259
138
Here we reuse the public cache, populate foo's private cache,
260
139
and populate the authenticated cache. The anonymous section is
261
140
uncached.
262
141
263
142
    >>> login('foo.bar@canonical.com')
264
143
    >>> print template(username='Foo Bar')
265
144
    Public:        Anonymous
266
145
    Private:       Foo Bar
267
146
    Anonymous:     Foo Bar
268
147
    Authenticated: Foo Bar
269
148
270
149
Here we reuse the public cache, populate test's private cache, and
271
150
reuse the authenticated cache. The anonymous section is uncached.
272
151
273
152
    >>> login('test@canonical.com')
274
153
    >>> print template(username='Test')
275
154
    Public:        Anonymous
276
155
    Private:       Test
277
156
    Anonymous:     Test
278
157
    Authenticated: Foo Bar
279
158
280
159
281
160
Nesting & Loops
282
161
---------------
283
162
284
163
Cached chunks can contain other cached chunks, useful for specifying
285
164
different timeouts of different visibilities.
286
165
287
166
    >>> template = TestPageTemplate(dedent("""\
288
167
    ...     <body tal:content="cache:private,25 seconds" tal:omit-tag="">
289
168
    ...         This bit is private to <span tal:replace="username" />
290
169
    ...         and cached up to 25 seconds, but contains
291
170
    ...         <span tal:content="cache:public,3 days" tal:omit-tag="">
292
171
    ...             this bit cached by <span tal:replace="username" />
293
172
    ...             which is public and cached up to 3 days.
294
173
    ...         </span>
295
174
    ...     </body>"""))
296
175
297
176
    >>> login('foo.bar@canonical.com')
298
177
    >>> print template(username="Foo Bar")
299
178
    This bit is private to Foo Bar and cached up to 25 seconds, but
300
179
    contains this bit cached by Foo Bar which is public and cached up
301
180
    to 3 days.
302
181
303
182
    >>> login('test@canonical.com')
304
183
    >>> print template(username="Test")
305
184
    This bit is private to Test and cached up to 25 seconds, but
306
185
    contains this bit cached by Foo Bar which is public and cached up
307
186
    to 3 days.
308
187
309
188
310
189
tal:repeat loops are fully supported. Each iteration of the loop gets
311
190
a different cache.
312
191
313
192
    >>> template = TestPageTemplate(dedent("""\
314
193
    ...     <body>
315
194
    ...         <div tal:repeat="i python:range(1,3)">
316
195
    ...             <div tal:replace="cache:public">
317
196
    ...                 <span tal:replace="param" />
318
197
    ...                 <span tal:replace="repeat/i/index" />
319
198
    ...             </div>
320
199
    ...         </div>
321
200
    ...     </body>"""))
322
201
    >>> print template(param='first')
323
202
    <body>
324
203
        <div> first 0 </div>
325
204
        <div> first 1 </div>
326
205
    </body>
327
206
328
207
    >>> print template(param='second')
329
208
    <body>
330
209
        <div> first 0 </div>
331
210
        <div> first 1 </div>
332
211
    </body>
333
212
334
213
335
214
336
0
215
337
=== modified file 'lib/lp/services/memcache/interfaces.py'
338
--- lib/lp/services/memcache/interfaces.py	2009-09-16 12:47:23 +0000
339
+++ lib/lp/services/memcache/interfaces.py	2010-03-06 08:12:37 +0000
340
@@ -4,7 +4,7 @@
341
4
"""Memcached interfaces."""
4
"""Memcached interfaces."""
342
5
5
343
6
__metaclass__ = type
6
__metaclass__ = type
345
7
__all__ = []
7
__all__ = ['IMemcacheClient']
346
8
8
347
9
from zope.interface import Interface
9
from zope.interface import Interface
348
10
10
349
11
11
350
=== added file 'lib/lp/services/memcache/tales.py'
351
--- lib/lp/services/memcache/tales.py	1970-01-01 00:00:00 +0000
352
+++ lib/lp/services/memcache/tales.py	2010-03-06 08:12:37 +0000
353
@@ -0,0 +1,294 @@
354
1
# Copyright 2010 Canonical Ltd.  This software is licensed under the
355
2
# GNU Affero General Public License version 3 (see the file LICENSE).
356
3
357
4
"""Implementation of the cache: namespace in TALES."""
358
5
359
6
__metaclass__ = type
360
7
__all__ = []
361
8
362
9
363
10
from hashlib import md5
364
11
import logging
365
12
import os.path
366
13
367
14
from zope.component import getUtility
368
15
from zope.interface import implements
369
16
from zope.tal.talinterpreter import TALInterpreter, I18nMessageTypes
370
17
from zope.tales.interfaces import ITALESExpression
371
18
372
19
from canonical.base import base
373
20
from canonical.config import config
374
21
from canonical.launchpad import versioninfo
375
22
from canonical.launchpad.webapp.interfaces import ILaunchBag
376
23
from lp.services.memcache.interfaces import IMemcacheClient
377
24
378
25
379
26
class MemcacheExpr:
380
27
    """Namespace to provide memcache caching of page template chunks.
381
28
382
29
    This namespace is exclusively used in tal:content directives.
383
30
    The only sensible way of using this is the following syntax:
384
31
385
32
    <div tal:content="cache:public, 1 hour">
386
33
        [... Potentially expensive page template chunk ...]
387
34
    </div>
388
35
    """
389
36
    implements(ITALESExpression)
390
37
    def __init__(self, name, expr, engine):
391
38
        """expr is in the format "visibility, 42 units".
392
39
393
40
        visibility is one of...
394
41
395
42
            public: All users see the same cached information.
396
43
397
44
            private: Authenticated users see a personal copy of the cached
398
45
                     information. Unauthenticated users share a copy of
399
46
                     the cached information.
400
47
401
48
            anonymous: Unauthenticated users use a shared copy of the
402
49
                       cached information. Authenticated users don't
403
50
                       use the cache. This probably isn't that useful
404
51
                       in practice, as Anonymous requests should already
405
52
                       be cached by reverse proxies on the production
406
53
                       systems.
407
54
408
55
            authenticated: Authenticated user share a copy of the cached
409
56
                           information, and unauthenticated users share
410
57
                           a seperate copy. Use this when information is
411
58
                           being hidden from unauthenticated users, eg.
412
59
                           for bug comments where email addresses are
413
60
                           obfuscated for unauthenticated users.
414
61
415
62
        units is one of 'seconds', 'minutes', 'hours' or 'days'.
416
63
417
64
        visibility is required. If the cache timeout is not specified,
418
65
        it defaults to 'never timeout' (memcache will still purge the
419
66
        information when in a LRU fashion when things fill up).
420
67
        """
421
68
        self._s = expr
422
69
423
70
        if ',' in expr:
424
71
            try:
425
72
                self.visibility, max_age = (s.strip() for s in expr.split(','))
426
73
            except ValueError:
427
74
                raise SyntaxError("Too many arguments in cache: expression")
428
75
        else:
429
76
            self.visibility = expr.strip()
430
77
            max_age = None
431
78
        assert self.visibility in (
432
79
            'anonymous', 'public', 'private', 'authenticated',
433
80
            ), 'visibility must be anonymous, public, private or authenticated'
434
81
435
82
        if max_age is None:
436
83
            self.max_age = 0
437
84
        else:
438
85
            try:
439
86
                value, unit = max_age.split(' ')
440
87
            except ValueError:
441
88
                raise SyntaxError(
442
89
                    "Unparsable age %s in cache: expression"
443
90
                    % repr(self.max_age))
444
91
            value = float(value)
445
92
            if unit[-1] == 's':
446
93
                unit = unit[:-1]
447
94
            if unit == 'second':
448
95
                pass
449
96
            elif unit == 'minute':
450
97
                value *= 60
451
98
            elif unit == 'hour':
452
99
                value *= 60 * 60
453
100
            elif unit == 'day':
454
101
                value *= 24 * 60 * 60
455
102
            else:
456
103
                raise AssertionError("Unknown unit %s" % unit)
457
104
            self.max_age = int(value)
458
105
459
106
    # For use with str.translate to sanitize keys. No control characters
460
107
    # allowed, and we skip ':' too since it is a magic separator.
461
108
    _key_translate_map = (
462
109
        '_'*33 + ''.join(chr(i) for i in range(33, ord(':'))) + '_'
463
110
        + ''.join(chr(i) for i in range(ord(':')+1, 127)) + '_' * 129)
464
111
465
112
    def getKey(self, econtext):
466
113
        """We need to calculate a unique key for this cached chunk.
467
114
468
115
        To ensure content is uniquely identified, we must include:
469
116
            - a user id if this chunk is not 'public'
470
117
            - the template source file name
471
118
            - the position in the source file
472
119
            - a counter to cope with cached chunks in loops
473
120
            - the revision number of the source tree
474
121
            - the config in use
475
122
            - the URL and query string
476
123
        """
477
124
        # We include the URL and query string in the key.
478
125
        # We use the full, unadulterated url to calculate a hash.
479
126
        # We use a sanitized version in the human readable chunk of
480
127
        # the key.
481
128
        request = econtext.getValue('request')
482
129
        url = str(request.URL) + '?' + str(request.get('QUERY_STRING', ''))
483
130
        url = url.encode('utf8') # Ensure it is a byte string.
484
131
        sanitized_url = url.translate(self._key_translate_map)
485
132
486
133
        # We include the source file and position in the source file in
487
134
        # the key.
488
135
        source_file = os.path.abspath(econtext.source_file)
489
136
        source_file = source_file[
490
137
            len(os.path.commonprefix([source_file, config.root + '/lib']))+1:]
491
138
492
139
        # We include the visibility in the key so private information
493
140
        # is not leaked. We use 'p' for public information, 'a' for
494
141
        # unauthenticated user information, 'l' for information shared
495
142
        # between all authenticated users, or ${Person.id} for private
496
143
        # information.
497
144
        if self.visibility == 'public':
498
145
            uid = 'p'
499
146
        else:
500
147
            logged_in_user = getUtility(ILaunchBag).user
501
148
            if logged_in_user is None:
502
149
                uid = 'a'
503
150
            elif self.visibility == 'authenticated':
504
151
                uid = 'l'
505
152
            else: # private visibility
506
153
                uid = str(logged_in_user.id)
507
154
508
155
        # We include a counter in the key, reset at the start of the
509
156
        # request, to ensure we get unique but repeatable keys inside
510
157
        # tal:repeat loops.
511
158
        counter_key = 'lp.services.memcache.tales.counter'
512
159
        counter = request.annotations.get(counter_key, 0) + 1
513
160
        request.annotations[counter_key] = counter
514
161
515
162
        # We use pt: as a unique prefix to ensure no clashes with other
516
163
        # components using the memcached servers. The order of components
517
164
        # below only matters for human readability and memcached reporting
518
165
        # tools - it doesn't really matter provided all the components are
519
166
        # included and separators used.
520
167
        key = "pt:%s:%s,%s:%s:%d,%d:%d,%s" % (
521
168
            config.instance_name,
522
169
            source_file, versioninfo.revno, uid,
523
170
            econtext.position[0], econtext.position[1], counter,
524
171
            sanitized_url,
525
172
            )
526
173
527
174
        # Memcached max key length is 250, so truncate but ensure uniqueness
528
175
        # with a hash. A short hash is good, provided it is still unique,
529
176
        # to preserve readability as much as possible. We include the
530
177
        # unsanitized URL in the hash to ensure uniqueness.
531
178
        key_hash = base(int(md5(key + url).hexdigest(), 16), 62)
532
179
        key = key[:250-len(key_hash)] + key_hash
533
180
534
181
        return key
535
182
536
183
    def __call__(self, econtext):
537
184
538
185
        # If we have an 'anonymous' visibility chunk and are logged in,
539
186
        # we don't cache. Return the 'default' magic token to interpret
540
187
        # the contents.
541
188
        request = econtext.getValue('request')
542
189
        if (self.visibility == 'anonymous'
543
190
            and getUtility(ILaunchBag).user is not None):
544
191
            return econtext.getDefault()
545
192
546
193
        # Calculate a unique key so we serve the right cached information.
547
194
        key = self.getKey(econtext)
548
195
549
196
        cached_chunk = getUtility(IMemcacheClient).get(key)
550
197
551
198
        if cached_chunk is None:
552
199
            logging.debug("Memcache miss for %s", key)
553
200
            return MemcacheMiss(key, self.max_age)
554
201
        else:
555
202
            logging.debug("Memcache hit for %s", key)
556
203
            return MemcacheHit(cached_chunk)
557
204
558
205
    def __str__(self):
559
206
        return 'memcache expression (%s)' % self._s
560
207
561
208
    def __repr__(self):
562
209
        return '<MemcacheExpr %s>' % self._s
563
210
564
211
565
212
class MemcacheMiss:
566
213
    """Callback for the customized TALInterpreter to invoke.
567
214
568
215
    If the memcache hit failed, the TALInterpreter interprets the
569
216
    tag contents and invokes this callback, which will store the
570
217
    result in memcache against the key calculated by the MemcacheExpr.
571
218
    """
572
219
    def __init__(self, key, max_age):
573
220
        self._key = key
574
221
        self._max_age = max_age
575
222
576
223
    def __call__(self, value):
577
224
        if getUtility(IMemcacheClient).set(
578
225
            self._key, value, self._max_age):
579
226
            logging.debug("Memcache set succeeded for %s", self._key)
580
227
        else:
581
228
            logging.warn("Memcache set failed for %s", self._key)
582
229
583
230
    def __repr__(self):
584
231
        return "<MemcacheCallback %s %d>" % (self._key, self._max_age)
585
232
586
233
587
234
class MemcacheHit:
588
235
    """A prerendered chunk retrieved from cache.
589
236
590
237
    We use a special object so the TALInterpreter knows that this
591
238
    information should not be quoted.
592
239
    """
593
240
    def __init__(self, value):
594
241
        self.value = value
595
242
596
243
597
244
# Oh my bleeding eyes! Monkey patching & cargo culting seems the sanest
598
245
# way of installing our extensions, which makes me sad.
599
246
600
247
def do_insertText_tal(self, stuff):
601
248
    text = self.engine.evaluateText(stuff[0])
602
249
    if text is None:
603
250
        return
604
251
    if text is self.Default:
605
252
        self.interpret(stuff[1])
606
253
        return
607
254
    # Start Launchpad customization
608
255
    if isinstance(text, MemcacheMiss):
609
256
        # We got a MemcacheCallback instance. This means we hit a
610
257
        # content="cache:..." attribute but there was no valid
611
258
        # data in memcache. So we need to interpret the enclosed
612
259
        # chunk of template and stuff it in the cache for next time.
613
260
        callback = text
614
261
        self.pushStream(self.StringIO())
615
262
        self.interpret(stuff[1])
616
263
        text = self.stream.getvalue()
617
264
        self.popStream()
618
265
        # Now we have generated the chunk, cache it for next time.
619
266
        callback(text)
620
267
        # And output it to the currently rendered page, unquoted.
621
268
        self.stream_write(text)
622
269
        return
623
270
    if isinstance(text, MemcacheHit):
624
271
        # Got a hit. Include the contents directly into the
625
272
        # rendered page, unquoted.
626
273
        self.stream_write(text.value)
627
274
        return
628
275
    # End Launchpad customization
629
276
    if isinstance(text, I18nMessageTypes):
630
277
        # Translate this now.
631
278
        text = self.translate(text)
632
279
    self._writeText(text)
633
280
TALInterpreter.bytecode_handlers_tal["insertText"] = do_insertText_tal
634
281
635
282
636
283
# Just like the original, except MemcacheHit and MemcacheMiss
637
284
# instances are also passed through unharmed.
638
285
def evaluateText(self, expr):
639
286
    text = self.evaluate(expr)
640
287
    if (text is None
641
288
        or isinstance(text, (basestring, MemcacheHit, MemcacheMiss))
642
289
        or text is self.getDefault()):
643
290
        return text
644
291
    return unicode(text)
645
292
import zope.pagetemplate.engine
646
293
zope.pagetemplate.engine.ZopeContextBase.evaluateText = evaluateText
647
294
648
0
295
649
=== added file 'lib/lp/services/memcache/tests/test_doc.py'
650
--- lib/lp/services/memcache/tests/test_doc.py	1970-01-01 00:00:00 +0000
651
+++ lib/lp/services/memcache/tests/test_doc.py	2010-03-06 08:12:37 +0000
652
@@ -0,0 +1,71 @@
653
1
# Copyright 2010 Canonical Ltd.  This software is licensed under the
654
2
# GNU Affero General Public License version 3 (see the file LICENSE).
655
3
656
4
"""Run doctests."""
657
5
658
6
__metaclass__ = type
659
7
660
8
import os.path
661
9
from textwrap import dedent
662
10
import unittest
663
11
664
12
from zope.component import getUtility
665
13
import zope.pagetemplate.engine
666
14
from zope.pagetemplate.pagetemplate import PageTemplate
667
15
from zope.publisher.browser import TestRequest
668
16
669
17
from canonical.launchpad.testing.systemdocs import (
670
18
    LayeredDocFileSuite, setUp, tearDown)
671
19
from canonical.testing.layers import LaunchpadFunctionalLayer, MemcachedLayer
672
20
from lp.services.memcache.interfaces import IMemcacheClient
673
21
from lp.services.testing import build_test_suite
674
22
from lp.testing import TestCase
675
23
676
24
677
25
here = os.path.dirname(os.path.realpath(__file__))
678
26
679
27
680
28
class TestPageTemplate(PageTemplate):
681
29
    """A cutdown PageTemplate implementation suitable for our tests."""
682
30
683
31
    _num_instances = 0
684
32
685
33
    def __init__(self, source):
686
34
        super(TestPageTemplate, self).__init__()
687
35
        TestPageTemplate._num_instances += 1
688
36
        self._my_instance_num = TestPageTemplate._num_instances
689
37
        self.pt_edit(source, 'text/html')
690
38
691
39
    def pt_source_file(self):
692
40
        return 'fake/test_%d.pt' % self._my_instance_num
693
41
694
42
    def pt_getEngine(self):
695
43
        # The <tales:expressiontype> ZCML only registers with this
696
44
        # engine, not the default.
697
45
        return zope.pagetemplate.engine.Engine
698
46
699
47
    def pt_getContext(self, args=(), options={}):
700
48
        # Build a minimal context. The cache: expression requires
701
49
        # a request.
702
50
        context = {'request': TestRequest()}
703
51
        context.update(options)
704
52
        return context
705
53
706
54
707
55
def memcacheSetUp(test):
708
56
    setUp(test)
709
57
    test.globs['TestPageTemplate'] = TestPageTemplate
710
58
    test.globs['dedent'] = dedent
711
59
    test.globs['MemcachedLayer'] = MemcachedLayer
712
60
713
61
714
62
special = {
715
63
    'tales-cache.txt': LayeredDocFileSuite(
716
64
        '../doc/tales-cache.txt',
717
65
        setUp=memcacheSetUp, tearDown=tearDown,
718
66
        layer=LaunchpadFunctionalLayer),
719
67
    }
720
68
721
69
722
70
def test_suite():
723
71
    return build_test_suite(here, special, layer=LaunchpadFunctionalLayer)
Reviewer	Review Type	Date Requested	Status
Gary Poster (community)		2010-02-26	Approve on 2010-02-26
Review via email: mp+20226@code.launchpad.net