desktopcouch

Merge lp:~cmiller/desktopcouch/attachments into lp:desktopcouch

  1. attachments
  2. Merge into trunk
Proposed by Chad Miller
Status: Merged
Approved by: Rodrigo Moya
Approved revision: not available
Merged at revision: not available
Proposed branch: lp:~cmiller/desktopcouch/attachments
Merge into: lp:desktopcouch
Diff against target: 274 lines (+166/-30)
3 files modified
desktopcouch/records/record.py (+64/-0)
desktopcouch/records/server_base.py (+34/-30)
desktopcouch/records/tests/test_server.py (+68/-0)
To merge this branch: bzr merge lp:~cmiller/desktopcouch/attachments
Reviewer Review Type Date Requested Status
Rodrigo Moya (community) Approve
Eric Casteleijn (community) Approve
Review via email: mp+15765@code.launchpad.net

Commit message

Make the database generate its own record IDs at "put" time, rather than let couchdb try to generate one. Quoth the python-couchdb folks, """The underlying HTTP ``POST`` method is not idempotent, and an automatic retry due to a problem somewhere on the networking stack may cause multiple documents being created in the database.""" A known ID solves the problem of lack of state.

Add support for attached documents. On Records, there are new methods,

  .attach(str_or_file, name, content_type)

  .detach(name)

  .list_attachments() --- list of strings

  .attachment_data(name) --- (string_blob, content_type)

To post a comment you must log in.
lp:~cmiller/desktopcouch/attachments updated
109. By Chad Miller

Raise more descriptive KeyErrors when accessing or manipulating attached
BLOBs.

Revision history for this message
Eric Casteleijn (thisfred) wrote :

Looks great, tests pass!

review: Approve
Revision history for this message
Rodrigo Moya (rodrigo-moya) wrote :

Looks good to me and all tests pass

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'desktopcouch/records/record.py'
2--- desktopcouch/records/record.py 2009-12-07 16:29:20 +0000
3+++ desktopcouch/records/record.py 2009-12-07 20:13:09 +0000
4@@ -85,12 +85,43 @@
5 return "Record type must be specified and should be a URL"
6
7
8+class Attachment(object):
9+ """This represents the value of the attachment. The name is stored as the key that points to this name, and the
10+
11+ We have a new BLOB to attach. Attachment(file_object, str_mime_type)
12+ We know the traits of a BLOB. Attachment()
13+ """
14+ def __init__(self, content=None, content_type=None):
15+ super(Attachment, self).__init__()
16+ if content is not None:
17+ if hasattr(content, "read") and hasattr(content, "seek"):
18+ pass # is a file-like object.
19+ elif isinstance(content, basestring):
20+ pass # is a string-like object.
21+ else:
22+ raise TypeError("Expected string or file (or None) as content.")
23+
24+ self.content = content
25+ self.content_type = content_type
26+ self.needs_synch = content is not None
27+
28+ def get_content_and_type(self):
29+ assert self.content is not None
30+ if hasattr(self.content, "read"):
31+ if hasattr(self.content, "seek"):
32+ self.content.seek(0, 0)
33+ return self.content.read(), self.content_type
34+ else:
35+ return self.content, self.content_type
36+
37+
38 class RecordData(object):
39 """Base class for all the Record data objects."""
40
41 def __init__(self, data):
42 validate(data)
43 self._data = data
44+ self._attachments = dict()
45
46 def __getitem__(self, key):
47 value = self._data[key]
48@@ -115,6 +146,39 @@
49 def __len__(self):
50 return len(self._data)
51
52+ def attach_by_reference(self, filename, getter_function):
53+ """This should only be called by the server code to refer to a BLOB
54+ that is in the database, so that we do not have to download it until
55+ the user wants it."""
56+ a = Attachment(None, None)
57+ a.get_content_and_type = getter_function
58+ self._attachments[filename] = a
59+
60+ def attach(self, content, filename, content_type):
61+ """Attach a file-like or string-like object, with a particular name and
62+ content type, to a document to be stored in the database. One can not
63+ clobber names that already exist."""
64+ if filename in self._attachments:
65+ raise KeyError("%r already exists" % (filename,))
66+ self._attachments[filename] = Attachment(content, content_type)
67+
68+ def detach(self, filename):
69+ """Remove a BLOB attached to a document."""
70+ try:
71+ self._attachments.pop(filename)
72+ except KeyError:
73+ raise KeyError("%r is not attached to this document" % (filename,))
74+
75+ def list_attachments(self):
76+ return self._attachments.keys()
77+
78+ def attachment_data(self, filename):
79+ """Retreive the attached data, the BLOB and content_type."""
80+ try:
81+ a = self._attachments[filename]
82+ except KeyError:
83+ raise KeyError("%r is not attached to this document" % (filename,))
84+ return a.get_content_and_type()
85
86 class RecordDict(RecordData):
87 """An object that represents an desktop CouchDB record fragment,
88
89=== modified file 'desktopcouch/records/server_base.py'
90--- desktopcouch/records/server_base.py 2009-11-30 16:03:01 +0000
91+++ desktopcouch/records/server_base.py 2009-12-07 20:13:09 +0000
92@@ -149,17 +149,41 @@
93 return None
94 data.update(couch_record)
95 record = self.record_factory(data=data)
96+
97+ def make_getter(source_db, document_id, attachment_name, content_type):
98+ """Closure storing the database for lower levels to use when needed."""
99+ def getter():
100+ return source_db.get_attachment(document_id, attachment_name), \
101+ content_type
102+ return getter
103+ if "_attachments" in data:
104+ for att_name, att_attributes in data["_attachments"].iteritems():
105+ record.attach_by_reference(att_name,
106+ make_getter(self.db, record_id, att_name,
107+ att_attributes["content_type"]))
108 return record
109
110 def put_record(self, record):
111 """Put a record in back end storage."""
112- record_id = record.record_id
113- record_data = record._data
114- if record_id:
115- self.db[record_id] = record_data
116- else:
117- record_id = self._add_record(record_data)
118- return record_id
119+ if not record.record_id:
120+ from uuid import uuid4 # Do not rely on couchdb to create an ID for us.
121+ record.record_id = uuid4().hex
122+ self.db[record.record_id] = record._data
123+
124+ # At this point, we've saved new document to the database, by we do not
125+ # know the revision number of it. We need *a* specific revision now,
126+ # so that we can attach BLOBs to it.
127+ #
128+ # This is bad. We can get the most recent revision, but that doesn't
129+ # assure us that what we're attaching records to is the revision we
130+ # just sent.
131+
132+ retreived_document = self.db[record.record_id]
133+ for attachment_name in record.list_attachments():
134+ data, content_type = record.attachment_data(attachment_name)
135+ self.db.put_attachment(retreived_document, data, attachment_name, content_type)
136+
137+ return record.record_id
138
139 def update_fields(self, record_id, fields):
140 """Safely update a number of fields. 'fields' being a
141@@ -175,10 +199,6 @@
142 continue
143 break
144
145- def _add_record(self, data):
146- """Add a new record to the storage backend."""
147- return self.db.create(data)
148-
149 def delete_record(self, record_id):
150 """Delete record with given id"""
151 record = self.db[record_id]
152@@ -361,29 +381,13 @@
153
154 or
155
156- >>> cb_id = glib.mainloop.idle_add(db_foo.report_changes, f)
157+ >>> # Make function that returns true, to signal never to remove.
158+ >>> report_forever = lambda **kw: db_foo.report_changes(**kw) or True
159+ >>> cb_id = glib.mainloop.idle_add(report_forever, f)
160
161 or
162
163 >>> task_id = twisted.task.looping_call(db_foo.report_changes, f)
164-
165- (
166- {
167- 'status': '200',
168- 'content-location': 'http://localhost:39535/test_view_add_and_delete/_changes',
169- 'transfer-encoding': 'chunked',
170- 'server': 'CouchDB/0.10.0 (Erlang OTP/R13B)',
171- 'cache-control': 'must-revalidate',
172- 'date': 'Tue, 17 Nov 2009 04:26:34 GMT',
173- 'content-type': 'application/json'
174- },
175- {
176- 'last_seq': 1,
177- 'results': [
178- {'changes': [{'rev': '1-6dfcf1344f23fe838db0bac6f319e67b'}], 'id': 'b5f39be14e161d46dc69e7120c6b36fa', 'seq': 1}
179- ]
180- }
181- )
182 """
183 now = time()
184 call_count = 0
185
186=== modified file 'desktopcouch/records/tests/test_server.py'
187--- desktopcouch/records/tests/test_server.py 2009-11-18 18:43:41 +0000
188+++ desktopcouch/records/tests/test_server.py 2009-12-07 20:13:09 +0000
189@@ -15,6 +15,7 @@
190 # along with desktopcouch. If not, see <http://www.gnu.org/licenses/>.
191 #
192 # Authors: Eric Casteleijn <eric.casteleijn@canonical.com>
193+# Chad Miller <chad.miller@canonical.com>
194
195 """testing database/contact.py module"""
196 import testtools
197@@ -24,6 +25,11 @@
198 from desktopcouch.records.server_base import row_is_deleted, NoSuchDatabase
199 from desktopcouch.records.record import Record
200
201+try:
202+ from io import StringIO
203+except ImportError:
204+ from cStringIO import StringIO as StringIO
205+
206 FAKE_RECORD_TYPE = "http://example.org/test"
207
208 js = """
209@@ -281,3 +287,65 @@
210 count = self.database.report_changes(rep)
211 self.assertEquals(0, count) # Ensure event count is zero.
212 self.assertEqual(saved_position, self.database._changes_since) # Pos'n is same.
213+
214+ def test_attachments(self):
215+ content = StringIO("0123456789\n==========\n\n" * 5)
216+
217+ constructed_record = Record({'record_number': 0}, record_type="http://example.com/")
218+
219+ # Before anything is attached, there are no attachments.
220+ self.assertEqual(constructed_record.list_attachments(), [])
221+
222+ # We can add attachments before a document is put in the DB.
223+ # Documents can look like files or strings.
224+ constructed_record.attach(content, "nu/mbe/rs", "text/plain")
225+ constructed_record.attach("string", "another document", "text/plain")
226+
227+ # We can read from a document that we constructed.
228+ out_file, out_content_type = \
229+ constructed_record.attachment_data("nu/mbe/rs")
230+ self.assertEqual(out_content_type, "text/plain")
231+
232+ # One can not put another document of the same name.
233+ self.assertRaises(KeyError, constructed_record.attach, content,
234+ "another document", "text/x-rst")
235+
236+ record_id = self.database.put_record(constructed_record)
237+ retrieved_record = self.database.get_record(record_id)
238+
239+ # We can add attachments after a document is put in the DB.
240+ retrieved_record.attach(content, "Document", "text/x-rst")
241+ record_id = self.database.put_record(retrieved_record) # push new version
242+ retrieved_record = self.database.get_record(record_id) # get new
243+
244+ # To replace, one must remove first.
245+ retrieved_record.detach("another document")
246+ retrieved_record.attach(content, "another document", "text/plain")
247+
248+ record_id = self.database.put_record(retrieved_record) # push new version
249+ retrieved_record = self.database.get_record(record_id) # get new
250+
251+ # We can get a list of attachments.
252+ self.assertEqual(set(retrieved_record.list_attachments()),
253+ set(["nu/mbe/rs", "Document", "another document"]))
254+
255+ # We can read from a document that we retrieved.
256+ out_data, out_content_type = retrieved_record.attachment_data("nu/mbe/rs")
257+ self.assertEqual(out_data, content.getvalue())
258+ self.assertEqual(out_content_type, "text/plain")
259+
260+ # Asking for a named document that does not exist causes KeyError.
261+ self.assertRaises(KeyError, retrieved_record.attachment_data,
262+ "NoExist")
263+ self.assertRaises(KeyError, constructed_record.attachment_data,
264+ "No Exist")
265+ self.assertRaises(KeyError, retrieved_record.detach,
266+ "NoExist")
267+
268+ for i, name in enumerate(retrieved_record.list_attachments()):
269+ if i != 0:
270+ retrieved_record.detach(name) # delete all but one.
271+ record_id = self.database.put_record(retrieved_record) # push new version.
272+
273+ # We can remove records with attachments.
274+ self.database.delete_record(record_id)

Subscribers

People subscribed via source and target branches