txAWS

Merge lp:~therve/txaws/ebs-support into lp:~txawsteam/txaws/trunk

  1. ebs-support
  2. Merge into trunk
Proposed by Thomas Herve
Status: Merged
Merged at revision: not available
Proposed branch: lp:~therve/txaws/ebs-support
Merge into: lp:~txawsteam/txaws/trunk
Diff against target: None lines
To merge this branch: bzr merge lp:~therve/txaws/ebs-support
Reviewer Review Type Date Requested Status
Duncan McGreggor Approve
Review via email: mp+10742@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Thomas Herve (therve) wrote :

This is ready to review in the attached branch. I also took the opportunity to remove the need of namespaces, as it feels useless to me.

Revision history for this message
Duncan McGreggor (oubiwann) wrote :

Looks great -- +1 for merge. Thanks for this new functionality!

review: Approve
lp:~therve/txaws/ebs-support updated
13. By Thomas Herve

Merge separate branches for EBS support.

14. By Thomas Herve

Add parameter support to describe volumes

15. By Thomas Herve

Add parameter support to describe snapshots

16. By Thomas Herve

Merge from trunk

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'txaws/ec2/client.py'
--- txaws/ec2/client.py 2009-08-21 03:26:35 +0000
+++ txaws/ec2/client.py 2009-08-26 13:50:25 +0000
@@ -3,7 +3,7 @@
33
4"""EC2 client support."""4"""EC2 client support."""
55
6from base64 import b64encode6from datetime import datetime
7from urllib import quote7from urllib import quote
88
9from twisted.web.client import getPage9from twisted.web.client import getPage
@@ -19,7 +19,7 @@
19 """An Amazon EC2 Reservation.19 """An Amazon EC2 Reservation.
2020
21 @attrib reservation_id: Unique ID of the reservation.21 @attrib reservation_id: Unique ID of the reservation.
22 @attrib owner_id: AWS Access Key ID of the user who owns the reservation. 22 @attrib owner_id: AWS Access Key ID of the user who owns the reservation.
23 @attrib groups: A list of security groups.23 @attrib groups: A list of security groups.
24 """24 """
25 def __init__(self, reservation_id, owner_id, groups=None):25 def __init__(self, reservation_id, owner_id, groups=None):
@@ -72,11 +72,32 @@
72 self.reservation = reservation72 self.reservation = reservation
7373
7474
75class Volume(object):
76 """An EBS volume instance."""
77
78 def __init__(self, id, size, status, create_time):
79 self.id = id
80 self.size = size
81 self.status = status
82 self.create_time = create_time
83 self.attachments = []
84
85
86class Attachment(object):
87 """An attachment of a L{Volume}."""
88
89 def __init__(self, instance_id, snapshot_id, availability_zone, status,
90 attach_time):
91 self.instance_id = instance_id
92 self.snapshot_id = snapshot_id
93 self.availability_zone = availability_zone
94 self.status = status
95 self.attach_time = attach_time
96
97
75class EC2Client(object):98class EC2Client(object):
76 """A client for EC2."""99 """A client for EC2."""
77100
78 name_space = '{http://ec2.amazonaws.com/doc/2008-12-01/}'
79
80 def __init__(self, creds=None, query_factory=None):101 def __init__(self, creds=None, query_factory=None):
81 """Create an EC2Client.102 """Create an EC2Client.
82103
@@ -102,7 +123,7 @@
102 """123 """
103 Parse the reservations XML payload that is returned from an AWS124 Parse the reservations XML payload that is returned from an AWS
104 describeInstances API call.125 describeInstances API call.
105 126
106 Instead of returning the reservations as the "top-most" object, we127 Instead of returning the reservations as the "top-most" object, we
107 return the object that most developers and their code will be128 return the object that most developers and their code will be
108 interested in: the instances. In instances reservation is available on129 interested in: the instances. In instances reservation is available on
@@ -111,53 +132,38 @@
111 root = XML(xml_bytes)132 root = XML(xml_bytes)
112 results = []133 results = []
113 # May be a more elegant way to do this:134 # May be a more elegant way to do this:
114 for reservation_data in root.find(self.name_space + 'reservationSet'):135 for reservation_data in root.find("reservationSet"):
115 # Get the security group information.136 # Get the security group information.
116 groups = []137 groups = []
117 for group_data in reservation_data.find(138 for group_data in reservation_data.find("groupSet"):
118 self.name_space + 'groupSet'):139 group_id = group_data.findtext("groupId")
119 group_id = group_data.findtext(self.name_space + 'groupId')
120 groups.append(group_id)140 groups.append(group_id)
121 # Create a reservation object with the parsed data.141 # Create a reservation object with the parsed data.
122 reservation = Reservation(142 reservation = Reservation(
123 reservation_id=reservation_data.findtext(143 reservation_id=reservation_data.findtext("reservationId"),
124 self.name_space + 'reservationId'),144 owner_id=reservation_data.findtext("ownerId"),
125 owner_id=reservation_data.findtext(
126 self.name_space + 'ownerId'),
127 groups=groups)145 groups=groups)
128 # Get the list of instances.146 # Get the list of instances.
129 instances = []147 instances = []
130 for instance_data in reservation_data.find(148 for instance_data in reservation_data.find("instancesSet"):
131 self.name_space + 'instancesSet'):149 instance_id = instance_data.findtext("instanceId")
132 instance_id = instance_data.findtext(
133 self.name_space + 'instanceId')
134 instance_state = instance_data.find(150 instance_state = instance_data.find(
135 self.name_space + 'instanceState').findtext(151 "instanceState").findtext("name")
136 self.name_space + 'name')152 instance_type = instance_data.findtext("instanceType")
137 instance_type = instance_data.findtext(153 image_id = instance_data.findtext("imageId")
138 self.name_space + 'instanceType')154 private_dns_name = instance_data.findtext("privateDnsName")
139 image_id = instance_data.findtext(self.name_space + 'imageId')155 dns_name = instance_data.findtext("dnsName")
140 private_dns_name = instance_data.findtext(156 key_name = instance_data.findtext("keyName")
141 self.name_space + 'privateDnsName')157 ami_launch_index = instance_data.findtext("amiLaunchIndex")
142 dns_name = instance_data.findtext(self.name_space + 'dnsName')158 launch_time = instance_data.findtext("launchTime")
143 key_name = instance_data.findtext(self.name_space + 'keyName')159 placement = instance_data.find("placement").findtext(
144 ami_launch_index = instance_data.findtext(160 "availabilityZone")
145 self.name_space + 'amiLaunchIndex')
146 launch_time = instance_data.findtext(
147 self.name_space + 'launchTime')
148 placement = instance_data.find(
149 self.name_space + 'placement').findtext(
150 self.name_space + 'availabilityZone')
151 products = []161 products = []
152 for product_data in instance_data.find(162 for product_data in instance_data.find("productCodesSet"):
153 self.name_space + 'productCodesSet'):163 product_code = product_data.findtext("productCode")
154 product_code = product_data.findtext(
155 self.name_space + 'productCode')
156 products.append(product_code)164 products.append(product_code)
157 kernel_id = instance_data.findtext(165 kernel_id = instance_data.findtext("kernelId")
158 self.name_space + 'kernelId')166 ramdisk_id = instance_data.findtext("ramdiskId")
159 ramdisk_id = instance_data.findtext(
160 self.name_space + 'ramdiskId')
161 instance = Instance(167 instance = Instance(
162 instance_id, instance_state, instance_type, image_id,168 instance_id, instance_state, instance_type, image_id,
163 private_dns_name, dns_name, key_name, ami_launch_index,169 private_dns_name, dns_name, key_name, ami_launch_index,
@@ -169,7 +175,7 @@
169175
170 def terminate_instances(self, *instance_ids):176 def terminate_instances(self, *instance_ids):
171 """Terminate some instances.177 """Terminate some instances.
172 178
173 @param instance_ids: The ids of the instances to terminate.179 @param instance_ids: The ids of the instances to terminate.
174 @return: A deferred which on success gives an iterable of180 @return: A deferred which on success gives an iterable of
175 (id, old-state, new-state) tuples.181 (id, old-state, new-state) tuples.
@@ -185,17 +191,48 @@
185 root = XML(xml_bytes)191 root = XML(xml_bytes)
186 result = []192 result = []
187 # May be a more elegant way to do this:193 # May be a more elegant way to do this:
188 for instance in root.find(self.name_space + 'instancesSet'):194 for instance in root.find("instancesSet"):
189 instanceId = instance.findtext(self.name_space + 'instanceId')195 instanceId = instance.findtext("instanceId")
190 previousState = instance.find(196 previousState = instance.find("previousState").findtext(
191 self.name_space + 'previousState').findtext(197 "name")
192 self.name_space + 'name')198 shutdownState = instance.find("shutdownState").findtext(
193 shutdownState = instance.find(199 "name")
194 self.name_space + 'shutdownState').findtext(
195 self.name_space + 'name')
196 result.append((instanceId, previousState, shutdownState))200 result.append((instanceId, previousState, shutdownState))
197 return result201 return result
198202
203 def describe_volumes(self):
204 """Describe available volumes."""
205 q = self.query_factory("DescribeVolumes", self.creds)
206 d = q.submit()
207 return d.addCallback(self._parse_volumes)
208
209 def _parse_volumes(self, xml_bytes):
210 root = XML(xml_bytes)
211 result = []
212 for volume_data in root.find("volumeSet"):
213 volume_id = volume_data.findtext("volumeId")
214 size = int(volume_data.findtext("size"))
215 status = volume_data.findtext("status")
216 create_time = volume_data.findtext("createTime")
217 create_time = datetime.strptime(
218 create_time[:19], "%Y-%m-%dT%H:%M:%S")
219 volume = Volume(volume_id, size, status, create_time)
220 result.append(volume)
221 for attachment_data in volume_data.find("attachmentSet"):
222 instance_id = attachment_data.findtext("instanceId")
223 snapshot_id = attachment_data.findtext("snapshotId")
224 availability_zone = attachment_data.findtext(
225 "availabilityZone")
226 status = attachment_data.findtext("status")
227 attach_time = attachment_data.findtext("attachTime")
228 attach_time = datetime.strptime(
229 attach_time[:19], "%Y-%m-%dT%H:%M:%S")
230 attachment = Attachment(
231 instance_id, snapshot_id, availability_zone, status,
232 attach_time)
233 volume.attachments.append(attachment)
234 return result
235
199236
200class Query(object):237class Query(object):
201 """A query that may be submitted to EC2."""238 """A query that may be submitted to EC2."""
@@ -204,7 +241,7 @@
204 """Create a Query to submit to EC2."""241 """Create a Query to submit to EC2."""
205 # Require params (2008-12-01 API):242 # Require params (2008-12-01 API):
206 # Version, SignatureVersion, SignatureMethod, Action, AWSAccessKeyId,243 # Version, SignatureVersion, SignatureMethod, Action, AWSAccessKeyId,
207 # Timestamp || Expires, Signature, 244 # Timestamp || Expires, Signature,
208 self.params = {'Version': '2008-12-01',245 self.params = {'Version': '2008-12-01',
209 'SignatureVersion': '2',246 'SignatureVersion': '2',
210 'SignatureMethod': 'HmacSHA1',247 'SignatureMethod': 'HmacSHA1',
@@ -242,7 +279,7 @@
242279
243 def sign(self):280 def sign(self):
244 """Sign this query using its built in credentials.281 """Sign this query using its built in credentials.
245 282
246 This prepares it to be sent, and should be done as the last step before283 This prepares it to be sent, and should be done as the last step before
247 submitting the query. Signing is done automatically - this is a public284 submitting the query. Signing is done automatically - this is a public
248 method to facilitate testing.285 method to facilitate testing.
249286
=== modified file 'txaws/ec2/tests/test_client.py'
--- txaws/ec2/tests/test_client.py 2009-08-21 03:26:35 +0000
+++ txaws/ec2/tests/test_client.py 2009-08-26 14:31:01 +0000
@@ -1,6 +1,7 @@
1# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>1# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
2# Licenced under the txaws licence available at /LICENSE in the txaws source.2# Licenced under the txaws licence available at /LICENSE in the txaws source.
33
4from datetime import datetime
4import os5import os
56
6from twisted.internet.defer import succeed7from twisted.internet.defer import succeed
@@ -86,6 +87,31 @@
86"""87"""
8788
8889
90sample_describe_volumes_result = """<?xml version="1.0"?>
91<DescribeVolumesResponse xmlns="http://ec2.amazonaws.com/doc/2008-12-01/">
92 <volumeSet>
93 <item>
94 <volumeId>vol-4282672b</volumeId>
95 <size>800</size>
96 <status>in-use</status>
97 <createTime>2008-05-07T11:51:50.000Z</createTime>
98 <attachmentSet>
99 <item>
100 <volumeId>vol-4282672b</volumeId>
101 <instanceId>i-6058a509</instanceId>
102 <size>800</size>
103 <snapshotId>snap-12345678</snapshotId>
104 <availabilityZone>us-east-1a</availabilityZone>
105 <status>attached</status>
106 <attachTime>2008-05-07T12:51:50.000Z</attachTime>
107 </item>
108 </attachmentSet>
109 </item>
110 </volumeSet>
111</DescribeVolumesResponse>
112"""
113
114
89class ReservationTestCase(TXAWSTestCase):115class ReservationTestCase(TXAWSTestCase):
90116
91 def test_reservation_creation(self):117 def test_reservation_creation(self):
@@ -118,7 +144,7 @@
118144
119145
120class TestEC2Client(TXAWSTestCase):146class TestEC2Client(TXAWSTestCase):
121 147
122 def test_init_no_creds(self):148 def test_init_no_creds(self):
123 os.environ['AWS_SECRET_ACCESS_KEY'] = 'foo'149 os.environ['AWS_SECRET_ACCESS_KEY'] = 'foo'
124 os.environ['AWS_ACCESS_KEY_ID'] = 'bar'150 os.environ['AWS_ACCESS_KEY_ID'] = 'bar'
@@ -160,7 +186,6 @@
160 self.assertEquals(instance.kernel_id, "aki-b51cf9dc")186 self.assertEquals(instance.kernel_id, "aki-b51cf9dc")
161 self.assertEquals(instance.ramdisk_id, "ari-b31cf9da")187 self.assertEquals(instance.ramdisk_id, "ari-b31cf9da")
162188
163
164 def test_parse_reservation(self):189 def test_parse_reservation(self):
165 ec2 = client.EC2Client(creds='foo')190 ec2 = client.EC2Client(creds='foo')
166 results = ec2._parse_instances(sample_describe_instances_result)191 results = ec2._parse_instances(sample_describe_instances_result)
@@ -286,3 +311,38 @@
286 query.sign()311 query.sign()
287 self.assertEqual('4hEtLuZo9i6kuG3TOXvRQNOrE/U=',312 self.assertEqual('4hEtLuZo9i6kuG3TOXvRQNOrE/U=',
288 query.params['Signature'])313 query.params['Signature'])
314
315
316class TestEBS(TXAWSTestCase):
317
318 def test_describe_volumes(self):
319
320 class StubQuery(object):
321 def __init__(stub, action, creds):
322 self.assertEqual(action, "DescribeVolumes")
323 self.assertEqual("foo", creds)
324
325 def submit(self):
326 return succeed(sample_describe_volumes_result)
327
328 def check_parsed_volumes(volumes):
329 self.assertEquals(len(volumes), 1)
330 volume = volumes[0]
331 self.assertEquals(volume.id, "vol-4282672b")
332 self.assertEquals(volume.size, 800)
333 self.assertEquals(volume.status, "in-use")
334 create_time = datetime(2008, 05, 07, 11, 51, 50)
335 self.assertEquals(volume.create_time, create_time)
336 self.assertEquals(len(volume.attachments), 1)
337 attachment = volume.attachments[0]
338 self.assertEquals(attachment.instance_id, "i-6058a509")
339 self.assertEquals(attachment.snapshot_id, "snap-12345678")
340 self.assertEquals(attachment.availability_zone, "us-east-1a")
341 self.assertEquals(attachment.status, "attached")
342 attach_time = datetime(2008, 05, 07, 12, 51, 50)
343 self.assertEquals(attachment.attach_time, attach_time)
344
345 ec2 = client.EC2Client(creds="foo", query_factory=StubQuery)
346 d = ec2.describe_volumes()
347 d.addCallback(check_parsed_volumes)
348 return d
289349
=== modified file 'txaws/storage/client.py'
--- txaws/storage/client.py 2009-08-20 12:15:12 +0000
+++ txaws/storage/client.py 2009-08-26 13:50:25 +0000
@@ -19,9 +19,6 @@
19from txaws.util import XML, calculate_md519from txaws.util import XML, calculate_md5
2020
2121
22name_space = '{http://s3.amazonaws.com/doc/2006-03-01/}'
23
24
25class S3Request(object):22class S3Request(object):
2623
27 def __init__(self, verb, bucket=None, object_name=None, data='',24 def __init__(self, verb, bucket=None, object_name=None, data='',
@@ -114,10 +111,10 @@
114 Parse XML bucket list response.111 Parse XML bucket list response.
115 """112 """
116 root = XML(response)113 root = XML(response)
117 for bucket in root.find(name_space + 'Buckets'):114 for bucket in root.find("Buckets"):
118 timeText = bucket.findtext(name_space + 'CreationDate')115 timeText = bucket.findtext("CreationDate")
119 yield {116 yield {
120 'name': bucket.findtext(name_space + 'Name'),117 'name': bucket.findtext("Name"),
121 'created': Time.fromISO8601TimeAndDate(timeText),118 'created': Time.fromISO8601TimeAndDate(timeText),
122 }119 }
123120
124121
=== modified file 'txaws/util.py'
--- txaws/util.py 2009-08-20 12:15:12 +0000
+++ txaws/util.py 2009-08-26 13:50:25 +0000
@@ -9,14 +9,14 @@
9import hmac9import hmac
10import time10import time
1111
12# Import XML from somwhere; here in one place to prevent duplication.12# Import XMLTreeBuilder from somwhere; here in one place to prevent duplication.
13try:13try:
14 from xml.etree.ElementTree import XML14 from xml.etree.ElementTree import XMLTreeBuilder
15except ImportError:15except ImportError:
16 from elementtree.ElementTree import XML16 from elementtree.ElementTree import XMLTreeBuilder
1717
1818
19__all__ = ['hmac_sha1', 'iso8601time']19__all__ = ["hmac_sha1", "iso8601time", "XML"]
2020
2121
22def calculate_md5(data):22def calculate_md5(data):
@@ -38,3 +38,17 @@
38 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time_tuple)38 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time_tuple)
39 else:39 else:
40 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())40 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
41
42
43class NamespaceFixXmlTreeBuilder(XMLTreeBuilder):
44
45 def _fixname(self, key):
46 if "}" in key:
47 key = key.split("}", 1)[1]
48 return key
49
50
51def XML(text):
52 parser = NamespaceFixXmlTreeBuilder()
53 parser.feed(text)
54 return parser.close()

Subscribers

People subscribed via source and target branches