Launchpad itself

Merge lp:~wgrant/launchpad/librarian-swift-unify into lp:launchpad

  1. librarian-swift-unify
  2. Merge into devel
Proposed by William Grant
Status: Merged
Merged at revision: 17878
Proposed branch: lp:~wgrant/launchpad/librarian-swift-unify
Merge into: lp:launchpad
Prerequisite: lp:~wgrant/launchpad/librarian-missing-500
Diff against target: 209 lines (+36/-105)
2 files modified
lib/lp/services/librarianserver/storage.py (+29/-26)
lib/lp/services/librarianserver/web.py (+7/-79)
To merge this branch: bzr merge lp:~wgrant/launchpad/librarian-swift-unify
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+281371@code.launchpad.net

Commit message

Drop the pre-Swift librarian codepaths. The Swift bits are battle-tested now.

Description of the change

Drop the pre-Swift librarian codepaths. The Swift bits are battle-tested now.

The feature flag check is now in LibrarianStorage.open.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/services/librarianserver/storage.py'
2--- lib/lp/services/librarianserver/storage.py 2015-01-09 07:36:08 +0000
3+++ lib/lp/services/librarianserver/storage.py 2015-12-27 04:05:56 +0000
4@@ -20,6 +20,7 @@
5 from lp.services.database import write_transaction
6 from lp.services.database.interfaces import IStore
7 from lp.services.database.postgresql import ConnectionString
8+from lp.services.features import getFeatureFlag
9 from lp.services.librarianserver import swift
10
11
12@@ -82,35 +83,37 @@
13
14 @defer.inlineCallbacks
15 def open(self, fileid):
16- # Log our attempt.
17- self.swift_download_attempts += 1
18-
19- if self.swift_download_attempts % 1000 == 0:
20- log.msg('{} Swift download attempts, {} failures'.format(
21- self.swift_download_attempts, self.swift_download_fails))
22-
23- # First, try and stream the file from Swift.
24- container, name = swift.swift_location(fileid)
25- swift_connection = swift.connection_pool.get()
26- try:
27- headers, chunks = yield deferToThread(
28- swift_connection.get_object,
29- container, name, resp_chunk_size=self.CHUNK_SIZE)
30- swift_stream = TxSwiftStream(swift_connection, chunks)
31- defer.returnValue(swift_stream)
32- except swiftclient.ClientException as x:
33- if x.http_status == 404:
34- swift.connection_pool.put(swift_connection)
35- else:
36+ if getFeatureFlag('librarian.swift.enabled'):
37+ # Log our attempt.
38+ self.swift_download_attempts += 1
39+
40+ if self.swift_download_attempts % 1000 == 0:
41+ log.msg('{} Swift download attempts, {} failures'.format(
42+ self.swift_download_attempts, self.swift_download_fails))
43+
44+ # First, try and stream the file from Swift.
45+ container, name = swift.swift_location(fileid)
46+ swift_connection = swift.connection_pool.get()
47+ try:
48+ headers, chunks = yield deferToThread(
49+ swift_connection.get_object,
50+ container, name, resp_chunk_size=self.CHUNK_SIZE)
51+ swift_stream = TxSwiftStream(swift_connection, chunks)
52+ defer.returnValue(swift_stream)
53+ except swiftclient.ClientException as x:
54+ if x.http_status == 404:
55+ swift.connection_pool.put(swift_connection)
56+ else:
57+ self.swift_download_fails += 1
58+ log.err(x)
59+ except Exception as x:
60 self.swift_download_fails += 1
61 log.err(x)
62- except Exception as x:
63- self.swift_download_fails += 1
64- log.err(x)
65+ # If Swift failed, for any reason, fall through to try and
66+ # stream the data from disk. In particular, files cannot be
67+ # found in Swift until librarian-feed-swift.py has put them
68+ # in there.
69
70- # If Swift failed, for any reason, try and stream the data from
71- # disk. In particular, files cannot be found in Swift until
72- # librarian-feed-swift.py has put them in there.
73 path = self._fileLocation(fileid)
74 if os.path.exists(path):
75 defer.returnValue(open(path, 'rb'))
76
77=== modified file 'lib/lp/services/librarianserver/web.py'
78--- lib/lp/services/librarianserver/web.py 2015-12-27 04:05:56 +0000
79+++ lib/lp/services/librarianserver/web.py 2015-12-27 04:05:56 +0000
80@@ -25,7 +25,6 @@
81 read_transaction,
82 write_transaction,
83 )
84-from lp.services.features import getFeatureFlag
85 from lp.services.librarian.client import url_path_quote
86 from lp.services.librarian.utils import guess_librarian_encoding
87
88@@ -118,17 +117,11 @@
89
90 token = request.args.get('token', [None])[0]
91 path = request.path
92- if getFeatureFlag('librarian.swift.enabled'):
93- deferred = deferToThread(
94- self._getFileAlias_swift, self.aliasID, token, path)
95- deferred.addCallback(
96- self._cb_getFileAlias_swift, filename, request)
97- else:
98- deferred = deferToThread(
99- self._getFileAlias, self.aliasID, token, path)
100- deferred.addCallback(
101- self._cb_getFileAlias, filename, request
102- )
103+ deferred = deferToThread(
104+ self._getFileAlias, self.aliasID, token, path)
105+ deferred.addCallback(
106+ self._cb_getFileAlias, filename, request
107+ )
108 deferred.addErrback(self._eb_getFileAlias)
109 return util.DeferredResource(deferred)
110
111@@ -137,15 +130,6 @@
112 try:
113 alias = self.storage.getFileAlias(aliasID, token, path)
114 return (alias.contentID, alias.filename,
115- alias.mimetype, alias.date_created, alias.restricted)
116- except LookupError:
117- raise NotFound
118-
119- @write_transaction
120- def _getFileAlias_swift(self, aliasID, token, path):
121- try:
122- alias = self.storage.getFileAlias(aliasID, token, path)
123- return (alias.contentID, alias.filename,
124 alias.mimetype, alias.date_created, alias.content.filesize,
125 alias.restricted)
126 except LookupError:
127@@ -162,43 +146,9 @@
128 else:
129 return fourOhFour
130
131+ @defer.inlineCallbacks
132 def _cb_getFileAlias(
133 self,
134- (dbcontentID, dbfilename, mimetype, date_created, restricted),
135- filename, request
136- ):
137- # Return a 404 if the filename in the URL is incorrect. This offers
138- # a crude form of access control (stuff we care about can have
139- # unguessable names effectively using the filename as a secret).
140- if dbfilename.encode('utf-8') != filename:
141- log.msg(
142- "404: dbfilename.encode('utf-8') != filename: %r != %r"
143- % (dbfilename.encode('utf-8'), filename))
144- return fourOhFour
145-
146- if self.storage.hasFile(dbcontentID):
147- # XXX: Brad Crittenden 2007-12-05 bug=174204: When encodings are
148- # stored as part of a file's metadata this logic will be replaced.
149- encoding, mimetype = guess_librarian_encoding(filename, mimetype)
150- # Set our caching headers. Public Librarian files can be
151- # cached forever, while private ones mustn't be at all.
152- request.setHeader(
153- 'Cache-Control',
154- 'max-age=31536000, public'
155- if not restricted else 'max-age=0, private')
156- return File(
157- mimetype, encoding, date_created,
158- self.storage._fileLocation(dbcontentID))
159- elif self.upstreamHost is not None:
160- return proxy.ReverseProxyResource(self.upstreamHost,
161- self.upstreamPort, request.path)
162- else:
163- raise AssertionError(
164- "Content %d missing from storage." % dbcontentID)
165-
166- @defer.inlineCallbacks
167- def _cb_getFileAlias_swift(
168- self,
169 (dbcontentID, dbfilename, mimetype, date_created, size,
170 restricted),
171 filename, request
172@@ -217,7 +167,7 @@
173 # XXX: Brad Crittenden 2007-12-05 bug=174204: When encodings are
174 # stored as part of a file's metadata this logic will be replaced.
175 encoding, mimetype = guess_librarian_encoding(filename, mimetype)
176- file = File_swift(mimetype, encoding, date_created, stream, size)
177+ file = File(mimetype, encoding, date_created, stream, size)
178 assert file.exists
179 # Set our caching headers. Public Librarian files can be
180 # cached forever, while private ones mustn't be at all.
181@@ -242,28 +192,6 @@
182 isLeaf = True
183
184 def __init__(
185- self, contentType, encoding, modification_time, *args, **kwargs):
186- # Have to convert the UTC datetime to POSIX timestamp (localtime)
187- offset = datetime.utcnow() - datetime.now()
188- local_modification_time = modification_time - offset
189- self._modification_time = time.mktime(
190- local_modification_time.timetuple())
191- static.File.__init__(self, *args, **kwargs)
192- self.type = contentType
193- self.encoding = encoding
194-
195- def getModificationTime(self):
196- """Override the time on disk with the time from the database.
197-
198- This is used by twisted to set the Last-Modified: header.
199- """
200- return self._modification_time
201-
202-
203-class File_swift(static.File):
204- isLeaf = True
205-
206- def __init__(
207 self, contentType, encoding, modification_time, stream, size):
208 # Have to convert the UTC datetime to POSIX timestamp (localtime)
209 offset = datetime.utcnow() - datetime.now()