pkgme service

Merge lp:~jml/pkgme-service/move-utils-from-tasks into lp:pkgme-service

  1. move-utils-from-tasks
  2. Merge into trunk
Proposed by Jonathan Lange
Status: Merged
Approved by: James Westby
Approved revision: 129
Merged at revision: 120
Proposed branch: lp:~jml/pkgme-service/move-utils-from-tasks
Merge into: lp:pkgme-service
Diff against target: 953 lines (+430/-372)
5 files modified
src/djpkgme/osutils.py (+120/-0)
src/djpkgme/tasks.py (+54/-150)
src/djpkgme/tests/__init__.py (+1/-0)
src/djpkgme/tests/test_osutils.py (+150/-0)
src/djpkgme/tests/test_tasks.py (+105/-222)
To merge this branch: bzr merge lp:~jml/pkgme-service/move-utils-from-tasks
Reviewer Review Type Date Requested Status
James Westby (community) Approve
Review via email: mp+124905@code.launchpad.net

Commit message

Move utilities out of tasks.py

Description of the change

I was going to add lzma support, but then realized that tasks.py is still
a huge mess.

This branch makes it slightly less messy by moving the tarball/zipfile stuff
somewhere else (osutils.py), and moving some of the stateless helper methods
to be functions.

To post a comment you must log in.
Revision history for this message
James Westby (james-w) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'src/djpkgme/osutils.py'
2--- src/djpkgme/osutils.py 1970-01-01 00:00:00 +0000
3+++ src/djpkgme/osutils.py 2012-09-18 12:37:20 +0000
4@@ -0,0 +1,120 @@
5+"""Tools for handling archives, and like things."""
6+
7+__all__ = [
8+ 'make_tarball',
9+ 'prepare_file',
10+ ]
11+
12+import errno
13+import itertools
14+import os
15+import shutil
16+import tarfile
17+import zipfile
18+
19+
20+def _try_extract_tarfile(path, target_dir):
21+ """Try extracting `path` to `target_dir` as a tarfile.
22+
23+ :return: True if successful, False if the file at `path` isn't
24+ a `tarfile`.
25+ """
26+ # XXX: untested.
27+ try:
28+ t = tarfile.open(path)
29+ except tarfile.ReadError:
30+ return False
31+ else:
32+ try:
33+ t.extractall(target_dir)
34+ finally:
35+ t.close()
36+ os.unlink(path)
37+ return True
38+
39+
40+def _try_extract_zipfile(path, target_dir):
41+ """Try extracting `path` to `target_dir` as a zipfile.
42+
43+ :return: True if successful, False if the file at `path` isn't
44+ a zipfile.
45+ """
46+ # XXX: untested.
47+ try:
48+ zf = zipfile.ZipFile(path, 'r')
49+ except zipfile.BadZipfile:
50+ return False
51+ else:
52+ try:
53+ zf.extractall(target_dir)
54+ finally:
55+ zf.close()
56+ os.unlink(path)
57+ return True
58+
59+
60+# The name of the directory that we create inside of the temporary build
61+# directory that pkgme does all of its work on.
62+DEFAULT_WORKING_DIRECTORY_NAME = 'working'
63+
64+def prepare_file(path, output_dir,
65+ working_dir_name=DEFAULT_WORKING_DIRECTORY_NAME):
66+ """Get 'path' ready for automated packaging inside 'output_dir'.
67+
68+ If 'path' is a tarball or zip file, then preparing it means extracting
69+ it into a new directory inside 'output_dir'.
70+
71+ If 'path' is a single file, like a PDF, then preparing it means
72+ copying it into a new directory inside 'output_dir'.
73+
74+ Either way, the key thing is that there is a new directory inside
75+ 'output_dir', inside which is the actual contents we want to try to
76+ automatically package.
77+
78+ :param path: The path where the uploaded file lives. Will be a
79+ tarball, PDF or something like that.
80+ :param output_dir: The directory in which to put the prepared version.
81+ :return: A path to a newly-created directory.
82+ """
83+ working_path = os.path.join(output_dir, working_dir_name)
84+ for method in [_try_extract_tarfile, _try_extract_zipfile]:
85+ if method(path, working_path):
86+ break
87+ else:
88+ os.makedirs(working_path)
89+ shutil.copy(path, working_path)
90+ return working_path
91+
92+
93+def make_tarball(source_dir, destination_dir, tarball_name):
94+ """Archive source_dir to destination_dir as a tarball.
95+
96+ Writes a tarball to $destination_dir/$tarball_name.tar.gz. If that
97+ exists, adds a number on the end to make sure we don't override an
98+ existing one.
99+
100+ :param source_dir: A directory that contains the thing to archive.
101+ :param destination_dir: The directory to write the tarball to.
102+ :param package_name: The name of the tarball.
103+ :return: the path of the created tarball.
104+ """
105+ try:
106+ os.makedirs(destination_dir, mode=0755)
107+ except OSError, e:
108+ if not (e.errno == errno.EEXIST
109+ and os.path.isdir(destination_dir)):
110+ raise
111+ destination = os.path.join(
112+ destination_dir, '%s.tar.gz' % (tarball_name,))
113+ disambiguator = itertools.count(1)
114+ while os.path.exists(destination):
115+ destination = os.path.join(
116+ destination_dir,
117+ '%s-%s.tar.gz' % (tarball_name, disambiguator.next()))
118+ t = tarfile.open(destination, 'w:gz')
119+ try:
120+ for name in os.listdir(source_dir):
121+ t.add(os.path.join(source_dir, name), name)
122+ finally:
123+ t.close()
124+ return destination
125
126=== modified file 'src/djpkgme/tasks.py'
127--- src/djpkgme/tasks.py 2012-09-14 13:59:48 +0000
128+++ src/djpkgme/tasks.py 2012-09-18 12:37:20 +0000
129@@ -1,12 +1,8 @@
130-import errno
131-import itertools
132 import json
133 import os
134 import shutil
135 import sys
136-import tarfile
137 import traceback
138-import zipfile
139
140 from bzrlib.urlutils import join as urljoin
141 from celery.registry import tasks
142@@ -36,6 +32,10 @@
143 submit_error,
144 submit_pkgme_info,
145 )
146+from .osutils import (
147+ make_tarball,
148+ prepare_file,
149+ )
150
151
152 class MissingFieldsError(Exception):
153@@ -119,6 +119,45 @@
154 return urljoin(settings.PKGME_OUTPUT_URL, filename)
155
156
157+def prepare_icons(working_dir, icons, logger=None):
158+ """Download the icons we are told about by the client.
159+
160+ :param icons: A dict mapping resolutions (e.g. '48x48' to URLs).
161+ :return: A dict mapping resolutions to local paths.
162+ """
163+ icon_files = {}
164+ for resolution, url in icons.items():
165+ if logger:
166+ logger.info("Downloading icon from %s", url)
167+ icon_files[resolution] = download_file(
168+ url, name=os.path.basename(url), working_dir=working_dir)
169+ if logger:
170+ logger.info("Downloaded icon from %s", url)
171+ return icon_files
172+
173+
174+def prepare_metadata(working_dir, metadata, logger=None):
175+ metadata = dict(metadata)
176+ icons = metadata.pop('icons', None)
177+ if icons:
178+ metadata[MetadataBackend.ICONS] = prepare_icons(
179+ working_dir, icons, logger)
180+ return metadata
181+
182+
183+def write_metadata(metadata, working_path):
184+ """Write metadata in a way that the binary backend expects.
185+
186+ :param metadata: Metadata from the client as a dictionary.
187+ :param working_path: The directory to store the metadata JSON file in.
188+ """
189+ metadata_filename = os.path.join(
190+ working_path, MetadataBackend.METADATA_FILE)
191+ with open(metadata_filename, 'w') as metadata_file:
192+ json.dump(as_pkgme_dict(metadata), metadata_file)
193+ return metadata_filename
194+
195+
196 class BuildPackageTask(Task):
197
198 # If one of these exceptions is raised when building packaging, we report
199@@ -135,142 +174,6 @@
200 'package_url',
201 ]
202
203- # The name of the directory that we create inside of the temporary build
204- # directory that pkgme does all of its work on.
205- WORKING_DIRECTORY_NAME = 'working'
206-
207- def _try_extract_tarfile(self, path, target_dir):
208- """Try extracting `path` to `target_dir` as a tarfile.
209-
210- :return: True if successful, False if the file at `path` isn't
211- a `tarfile`.
212- """
213- try:
214- t = tarfile.open(path)
215- except tarfile.ReadError:
216- return False
217- else:
218- try:
219- t.extractall(target_dir)
220- finally:
221- t.close()
222- os.unlink(path)
223- return True
224-
225- def _try_extract_zipfile(self, path, target_dir):
226- """Try extracting `path` to `target_dir` as a zipfile.
227-
228- :return: True if successful, False if the file at `path` isn't
229- a zipfile.
230- """
231- try:
232- zf = zipfile.ZipFile(path, 'r')
233- except zipfile.BadZipfile:
234- return False
235- else:
236- try:
237- zf.extractall(target_dir)
238- finally:
239- zf.close()
240- os.unlink(path)
241- return True
242-
243- def prepare_file(self, path, output_dir):
244- """Get 'path' ready for automated packaging inside 'output_dir'.
245-
246- If 'path' is a tarball or zip file, then preparing it means extracting
247- it into a new directory inside 'output_dir'.
248-
249- If 'path' is a single file, like a PDF, then preparing it means
250- copying it into a new directory inside 'output_dir'.
251-
252- Either way, the key thing is that there is a new directory inside
253- 'output_dir', inside which is the actual contents we want to try to
254- automatically package.
255-
256- :param path: The path where the uploaded file lives. Will be a
257- tarball, PDF or something like that.
258- :param output_dir: The directory in which to put the prepared version.
259- :return: A path to a newly-created directory.
260- """
261- working_path = os.path.join(output_dir, self.WORKING_DIRECTORY_NAME)
262- for method in (self._try_extract_tarfile, self._try_extract_zipfile):
263- if method(path, working_path):
264- break
265- else:
266- os.makedirs(working_path)
267- shutil.copy(path, working_path)
268- return working_path
269-
270- def prepare_icons(self, working_dir, icons):
271- """Download the icons we are told about by the client.
272-
273- :param icons: A dict mapping resolutions (e.g. '48x48' to URLs).
274- :return: A dict mapping resolutions to local paths.
275- """
276- icon_files = {}
277- for resolution, url in icons.items():
278- self.get_logger().info("Downloading icon from %s", url)
279- icon_files[resolution] = download_file(
280- url, name=os.path.basename(url), working_dir=working_dir)
281- self.get_logger().info("Downloaded icon from %s", url)
282- return icon_files
283-
284- def prepare_metadata(self, working_dir, metadata):
285- metadata = dict(metadata)
286- icons = metadata.pop('icons', None)
287- if icons:
288- metadata[MetadataBackend.ICONS] = self.prepare_icons(
289- working_dir, icons)
290- return metadata
291-
292- def write_metadata(self, metadata, working_path):
293- """Write metadata in a way that the binary backend expects.
294-
295- :param metadata: Metadata from the client as a dictionary.
296- :param working_path: The directory to store the metadata JSON file in.
297- """
298- metadata_filename = os.path.join(
299- working_path, MetadataBackend.METADATA_FILE)
300- with open(metadata_filename, 'w') as metadata_file:
301- json.dump(as_pkgme_dict(metadata), metadata_file)
302- return metadata_filename
303-
304- def save_package(self, source_package_location, destination_dir,
305- package_name):
306- """Save the source package to destination_dir as a tarball.
307-
308- Writes a tarball to $destination_dir/$package_name.tar.gz. If that
309- exists, adds a number on the end to make sure we don't override an
310- existing one.
311-
312- :param source_package_location: A directory that contains the built
313- source package.
314- :param destination_dir: The directory to write the tarball to.
315- :param package_name: The name of the source package. Will be used
316- as part of the tarball name.
317- """
318- try:
319- os.makedirs(destination_dir, mode=0755)
320- except OSError, e:
321- if not (e.errno == errno.EEXIST
322- and os.path.isdir(destination_dir)):
323- raise
324- destination = os.path.join(
325- destination_dir, '%s.tar.gz' % (package_name,))
326- disambiguator = itertools.count(1)
327- while os.path.exists(destination):
328- destination = os.path.join(
329- destination_dir,
330- '%s-%s.tar.gz' % (package_name, disambiguator.next()))
331- t = tarfile.open(destination, 'w:gz')
332- try:
333- for name in os.listdir(source_package_location):
334- t.add(os.path.join(source_package_location, name), name)
335- finally:
336- t.close()
337- return destination
338-
339 def build_package(self, metadata, overrides):
340 """Build packaging for 'metadata'.
341
342@@ -300,34 +203,35 @@
343 # devportal-metadata.json) are tarred up and stored in a preconfigured
344 # location: PKGME_OUTPUT_DIRECTORY.
345 working_dirs = []
346+ logger = self.get_logger()
347 with TempDir() as temp_dir:
348 output_dir = temp_dir.path
349 download_dir = os.path.join(
350 output_dir, self.DOWNLOAD_DIRECTORY_NAME)
351 os.mkdir(download_dir)
352 working_dirs.append(download_dir)
353- self.get_logger().info(
354+ logger.info(
355 "Downloading package from %s", metadata['package_url'])
356 download_path = download_file(
357 metadata['package_url'], download_dir)
358- self.get_logger().info(
359+ logger.info(
360 "Downloaded package from %s", metadata['package_url'])
361- working_path = self.prepare_file(download_path, output_dir)
362+ working_path = prepare_file(download_path, output_dir)
363 # XXX: Missing tests to show that the downloaded icons don't
364 # appear in the saved tarball.
365 icons_dir = os.path.join(output_dir, 'icons')
366 os.mkdir(icons_dir)
367 working_dirs.append(icons_dir)
368- metadata = self.prepare_metadata(icons_dir, metadata)
369- metadata_filename = self.write_metadata(metadata, working_path)
370+ metadata = prepare_metadata(icons_dir, metadata, logger)
371+ metadata_filename = write_metadata(metadata, working_path)
372 # Run pkgme: generate the packaging and build a source package.
373- self.get_logger().info("Running pkgme")
374+ logger.info("Running pkgme")
375 backends = load_backends()
376 pkgme_decision = get_all_info(
377 working_path, backends, settings.PKGME_ALLOWED_BACKENDS)
378 pkgme_info = pkgme_decision['info']
379 write_packaging_info(working_path, pkgme_info)
380- self.get_logger().info("pkgme completed")
381+ logger.info("pkgme completed")
382 # XXX: This duplicates work done by write_packaging. However, we
383 # need the package_name in order to create the tarball in a way
384 # that looks good.
385@@ -344,14 +248,14 @@
386 new_working_path = os.path.join(output_dir, package_name)
387 os.rename(working_path, new_working_path)
388 working_dirs.append(new_working_path)
389- self.get_logger().info("Building source package")
390+ logger.info("Building source package")
391 build_package(new_working_path, False)
392- self.get_logger().info("Finished building source package")
393+ logger.info("Finished building source package")
394 # Create a new tarball containing the source package and the
395 # metadata file, but not the pkgme working set.
396 for working_dir in working_dirs:
397 shutil.rmtree(working_dir)
398- return self.save_package(
399+ return make_tarball(
400 output_dir, settings.PKGME_OUTPUT_DIRECTORY, package_name)
401
402 def _submit_error(self, metadata, error_data):
403
404=== modified file 'src/djpkgme/tests/__init__.py'
405--- src/djpkgme/tests/__init__.py 2012-07-23 14:15:09 +0000
406+++ src/djpkgme/tests/__init__.py 2012-09-18 12:37:20 +0000
407@@ -1,5 +1,6 @@
408 from .test_handlers import *
409 from .test_integration import *
410+from .test_osutils import *
411 from .test_preflight import *
412 from .test_tasks import *
413 from .test_version import *
414
415=== added file 'src/djpkgme/tests/test_osutils.py'
416--- src/djpkgme/tests/test_osutils.py 1970-01-01 00:00:00 +0000
417+++ src/djpkgme/tests/test_osutils.py 2012-09-18 12:37:20 +0000
418@@ -0,0 +1,150 @@
419+import errno
420+import os
421+import tarfile
422+import zipfile
423+
424+from fixtures import TempDir
425+from testtools import TestCase
426+from testtools.matchers import (
427+ DirExists,
428+ FileContains,
429+ HasPermissions,
430+ TarballContains,
431+ )
432+
433+from ..osutils import (
434+ DEFAULT_WORKING_DIRECTORY_NAME,
435+ make_tarball,
436+ prepare_file,
437+ )
438+from .factory import PkgmeObjectFactory
439+
440+
441+class TestPrepareFile(TestCase):
442+
443+ def test_extracts_tarball(self):
444+ # If the file downloaded is a tarball then it is extracted
445+ factory = self.useFixture(PkgmeObjectFactory())
446+ path = factory.make_tarball(self.useFixture(TempDir()).path)
447+ tarball = tarfile.open(path)
448+ top_level_dir = tarball.getnames()[0]
449+ tempdir = self.useFixture(TempDir()).path
450+ new_path = prepare_file(path, tempdir)
451+ self.assertEqual(
452+ os.path.join(tempdir, DEFAULT_WORKING_DIRECTORY_NAME),
453+ new_path)
454+ self.assertThat(new_path, DirExists())
455+ self.assertThat(os.path.join(new_path, top_level_dir), DirExists())
456+
457+ def test_extracts_zips(self):
458+ # If the file downloaded is a .zip then it is extracted
459+ top_level_dir = 'top_level'
460+ tempdir = self.useFixture(TempDir()).path
461+ path = os.path.join(tempdir, 'application.zip')
462+ zf = zipfile.ZipFile(path, 'w')
463+ try:
464+ zf.writestr(os.path.join(top_level_dir, 'afile'), 'some content\n')
465+ finally:
466+ zf.close()
467+ new_path = prepare_file(path, tempdir)
468+ self.assertEqual(
469+ os.path.join(tempdir, DEFAULT_WORKING_DIRECTORY_NAME),
470+ new_path)
471+ self.assertThat(new_path, DirExists())
472+ self.assertThat(os.path.join(new_path, top_level_dir), DirExists())
473+
474+ def test_copies_non_tarball(self):
475+ # If the file downloaded is not a tarball then it is copied
476+ source_dir = self.useFixture(TempDir()).path
477+ filename = 'afile'
478+ content = 'some content\n'
479+ with open(os.path.join(source_dir, filename), 'w') as f:
480+ f.write(content)
481+ tempdir = self.useFixture(TempDir()).path
482+ new_path = prepare_file(
483+ os.path.join(source_dir, filename), tempdir)
484+ self.assertEqual(
485+ os.path.join(tempdir, DEFAULT_WORKING_DIRECTORY_NAME),
486+ new_path)
487+ self.assertThat(new_path, DirExists())
488+ self.assertThat(os.path.join(new_path, filename),
489+ FileContains(content))
490+
491+
492+class TestMakeTarball(TestCase):
493+
494+ def test_destination_multiple_parents_dont_exist(self):
495+ # make_tarball creates the destination directory if it doesn't already
496+ # exist.
497+ factory = self.useFixture(PkgmeObjectFactory())
498+ metadata = factory.make_metadata()
499+ package_name = metadata['package_name']
500+ path = self.useFixture(TempDir()).path
501+ destination_dir = os.path.join(path, 'does', 'not', 'exist')
502+ make_tarball(path, destination_dir, package_name)
503+ self.assertThat(destination_dir, DirExists())
504+ self.assertThat(destination_dir, HasPermissions('0755'))
505+
506+ def test_destination_is_file(self):
507+ # If the destination directory is actually a file, we raise an error.
508+ factory = self.useFixture(PkgmeObjectFactory())
509+ metadata = factory.make_metadata()
510+ package_name = metadata['package_name']
511+ path = self.useFixture(TempDir()).path
512+ destination_dir = os.path.join(path, 'some-file')
513+ with open(destination_dir, 'w') as fp:
514+ fp.write('foo')
515+ e = self.assertRaises(
516+ OSError,
517+ make_tarball, path, destination_dir, package_name)
518+ self.assertEqual(errno.EEXIST, e.errno)
519+
520+ def test_destination_exists(self):
521+ # If the destination directory already exists, we proceed normally.
522+ factory = self.useFixture(PkgmeObjectFactory())
523+ metadata = factory.make_metadata()
524+ package_name = metadata['package_name']
525+ path = self.useFixture(TempDir()).path
526+ destination_dir = self.useFixture(TempDir()).path
527+ make_tarball(path, destination_dir, package_name)
528+ self.assertThat(destination_dir, DirExists())
529+
530+ def test_creates_tarball(self):
531+ # make_tarball creates a tarball in the destination directory that's
532+ # named after the package and contains whatever was in the source
533+ # directory.
534+ factory = self.useFixture(PkgmeObjectFactory())
535+ metadata = factory.make_metadata()
536+ package_name = metadata['package_name']
537+ path = self.useFixture(TempDir()).path
538+ os.mkdir(os.path.join(path, 'a-directory'))
539+ with open(os.path.join(path, 'a-file'), 'w') as fp:
540+ fp.write('foo')
541+ destination_dir = self.useFixture(TempDir()).path
542+ destination = make_tarball(path, destination_dir, package_name)
543+ self.assertEqual(
544+ os.path.join(destination_dir, '%s.tar.gz' % (package_name,)),
545+ destination)
546+ self.assertThat(
547+ destination,
548+ TarballContains(['a-directory', 'a-file']))
549+
550+ def test_uniquifies(self):
551+ # If package-name.tar.gz exists, try package-name-1.tar.gz, etc.
552+ factory = self.useFixture(PkgmeObjectFactory())
553+ metadata = factory.make_metadata()
554+ package_name = metadata['package_name']
555+ path = self.useFixture(TempDir()).path
556+ os.mkdir(os.path.join(path, 'a-directory'))
557+ with open(os.path.join(path, 'a-file'), 'w') as fp:
558+ fp.write('foo')
559+ destination_dir = self.useFixture(TempDir()).path
560+ make_tarball(path, destination_dir, package_name)
561+ destination = make_tarball(path, destination_dir, package_name)
562+ self.assertEqual(
563+ destination,
564+ os.path.join(destination_dir, '%s-1.tar.gz' % (package_name,)))
565+ destination = make_tarball(path, destination_dir, package_name)
566+ self.assertEqual(
567+ destination,
568+ os.path.join(destination_dir, '%s-2.tar.gz' % (package_name,)))
569
570=== modified file 'src/djpkgme/tests/test_tasks.py'
571--- src/djpkgme/tests/test_tasks.py 2012-09-14 14:04:48 +0000
572+++ src/djpkgme/tests/test_tasks.py 2012-09-18 12:37:20 +0000
573@@ -1,13 +1,11 @@
574 # Run tasks eagerly for tests. Taken from
575 # http://ask.github.com/celery/cookbook/unit-testing.html
576
577-import errno
578 import json
579 import os
580 import sys
581 import tarfile
582 import traceback
583-import zipfile
584
585 from devportalbinary.metadata import MetadataBackend
586 from django.conf import settings
587@@ -21,11 +19,9 @@
588 DirExists,
589 Equals,
590 FileContains,
591- HasPermissions,
592 MatchesListwise,
593 SamePath,
594 StartsWith,
595- TarballContains,
596 )
597 from testtools import TestCase
598
599@@ -36,10 +32,14 @@
600 BuildPackageTask,
601 CallbackError,
602 get_url_for_file,
603+ prepare_icons,
604+ prepare_metadata,
605 translate_dict,
606+ write_metadata,
607 )
608 from ..client import get_response_dict
609 from djpkgme.tests.factory import (
610+ PkgmeObjectFactory,
611 TestCaseWithFactory,
612 )
613 from djpkgme.tests.helpers import FakeResponse
614@@ -120,6 +120,107 @@
615 self.assertEqual(expected, as_pkgme_dict(metadata))
616
617
618+class TestExcInfoAsDict(TestCase):
619+
620+ def test_exc_info_as_dict(self):
621+ try:
622+ raise RuntimeError('whatever')
623+ except RuntimeError:
624+ exc_info = sys.exc_info()
625+ error_dict = tasks.exc_info_as_dict(exc_info)
626+ self.assertEqual(
627+ {'type': 'RuntimeError',
628+ 'traceback': ''.join(traceback.format_exception(*exc_info)),
629+ }, error_dict)
630+
631+
632+class TestWriteMetadata(TestCase):
633+
634+ def test_write_metadata(self):
635+ factory = self.useFixture(PkgmeObjectFactory())
636+ metadata = factory.make_metadata()
637+ temp_dir = self.useFixture(TempDir()).path
638+ metadata_filename = write_metadata(metadata, temp_dir)
639+ self.assertThat(
640+ metadata_filename,
641+ SamePath(os.path.join(temp_dir, MetadataBackend.METADATA_FILE)))
642+ self.assertThat(
643+ metadata_filename,
644+ FileContains(json.dumps(as_pkgme_dict(metadata))))
645+
646+
647+class TestPrepareIcons(TestCase):
648+
649+ @patch('djpkgme.tasks.download_file')
650+ def test_downloads_icons(self, mock_download):
651+ icons = {
652+ '48x48': 'http://whatever/something.png',
653+ '64x64': 'http://whatever/foo/another.jpg',
654+ }
655+ path = self.useFixture(TempDir()).path
656+ prepare_icons(path, icons)
657+ self.assertEqual(
658+ sorted(
659+ [mock.call(
660+ 'http://whatever/something.png',
661+ name='something.png', working_dir=path),
662+ mock.call(
663+ 'http://whatever/foo/another.jpg',
664+ name='another.jpg', working_dir=path),
665+ ]),
666+ sorted(mock_download.call_args_list))
667+
668+ def test_returns_paths(self):
669+ def download_file(url, name, working_dir):
670+ return '%s/++download++/%s' % (working_dir, name)
671+ self.patch(tasks, 'download_file', download_file)
672+ icons = {
673+ '48x48': 'http://whatever/something.png',
674+ '64x64': 'http://whatever/foo/another.jpg',
675+ }
676+ path = self.useFixture(TempDir()).path
677+ new_icons = prepare_icons(path, icons)
678+ self.assertEqual(
679+ {'48x48': '%s/++download++/something.png' % (path,),
680+ '64x64': '%s/++download++/another.jpg' % (path,),
681+ }, new_icons)
682+
683+
684+class TestPrepareMetadata(TestCase):
685+
686+ def test_default(self):
687+ path = self.useFixture(TempDir()).path
688+ factory = self.useFixture(PkgmeObjectFactory())
689+ metadata = factory.make_metadata()
690+ prepared = prepare_metadata(path, metadata)
691+ self.assertEqual(prepared, metadata)
692+
693+ def test_maintainer_provided(self):
694+ path = self.useFixture(TempDir()).path
695+ factory = self.useFixture(PkgmeObjectFactory())
696+ metadata = factory.make_metadata()
697+ metadata[MetadataBackend.MAINTAINER] = (
698+ 'Upstream <upstream@example.com>')
699+ prepared = prepare_metadata(path, metadata)
700+ self.assertEqual(prepared, metadata)
701+
702+ @patch('djpkgme.tasks.prepare_icons')
703+ def test_icons(self, prepare_icons):
704+ prepare_icons.return_value = {'48x48': '/tmp/whatever.png'}
705+ path = self.useFixture(TempDir()).path
706+ factory = self.useFixture(PkgmeObjectFactory())
707+ metadata = factory.make_metadata()
708+ expected = dict(metadata)
709+ metadata['icons'] = {
710+ '48x48': 'http://example.com/whatever.png',
711+ }
712+ expected[MetadataBackend.ICONS] = {
713+ '48x48': '/tmp/whatever.png',
714+ }
715+ prepared = prepare_metadata(path, metadata)
716+ self.assertEqual(expected, prepared)
717+
718+
719 class BuildPackageTaskTestCase(TestCaseWithFactory):
720
721 @patch('httplib2.Http.request', return_value=(FakeResponse(200, 'OK'), ''))
722@@ -157,144 +258,6 @@
723 self.assertEqual(args, (metadata['callback_url'],))
724 self.assertEqual(expected, json.loads(kwargs['body']))
725
726- def test_write_metadata(self):
727- metadata = self.factory.make_metadata()
728- task = BuildPackageTask()
729- temp_dir = self.useFixture(TempDir()).path
730- metadata_filename = task.write_metadata(metadata, temp_dir)
731- self.assertThat(
732- metadata_filename,
733- SamePath(os.path.join(temp_dir, MetadataBackend.METADATA_FILE)))
734- self.assertThat(
735- metadata_filename,
736- FileContains(json.dumps(as_pkgme_dict(metadata))))
737-
738- def test_prepare_file_extracts_tarball(self):
739- # If the file downloaded is a tarball then it is extracted
740- task = BuildPackageTask()
741- path = self.factory.make_tarball(self.useFixture(TempDir()).path)
742- tarball = tarfile.open(path)
743- top_level_dir = tarball.getnames()[0]
744- tempdir = self.useFixture(TempDir()).path
745- new_path = task.prepare_file(path, tempdir)
746- self.assertEqual(
747- os.path.join(tempdir, BuildPackageTask.WORKING_DIRECTORY_NAME),
748- new_path)
749- self.assertThat(new_path, DirExists())
750- self.assertThat(os.path.join(new_path, top_level_dir), DirExists())
751-
752- def test_prepare_file_extracts_zips(self):
753- # If the file downloaded is a .zip then it is extracted
754- task = BuildPackageTask()
755- top_level_dir = 'top_level'
756- tempdir = self.useFixture(TempDir()).path
757- path = os.path.join(tempdir, 'application.zip')
758- zf = zipfile.ZipFile(path, 'w')
759- try:
760- zf.writestr(os.path.join(top_level_dir, 'afile'), 'some content\n')
761- finally:
762- zf.close()
763- new_path = task.prepare_file(path, tempdir)
764- self.assertEqual(
765- os.path.join(tempdir, BuildPackageTask.WORKING_DIRECTORY_NAME),
766- new_path)
767- self.assertThat(new_path, DirExists())
768- self.assertThat(os.path.join(new_path, top_level_dir), DirExists())
769-
770- def test_prepare_file_copies_non_tarball(self):
771- # If the file downloaded is not a tarball then it is copied
772- task = BuildPackageTask()
773- source_dir = self.useFixture(TempDir()).path
774- filename = 'afile'
775- content = 'some content\n'
776- with open(os.path.join(source_dir, filename), 'w') as f:
777- f.write(content)
778- tempdir = self.useFixture(TempDir()).path
779- new_path = task.prepare_file(
780- os.path.join(source_dir, filename), tempdir)
781- self.assertEqual(
782- os.path.join(tempdir, BuildPackageTask.WORKING_DIRECTORY_NAME),
783- new_path)
784- self.assertThat(new_path, DirExists())
785- self.assertThat(os.path.join(new_path, filename),
786- FileContains(content))
787-
788- def test_save_package_destination_multiple_parents_dont_exist(self):
789- # save_package creates the destination directory if it doesn't already
790- # exist.
791- metadata = self.factory.make_metadata()
792- package_name = metadata['package_name']
793- task = BuildPackageTask()
794- path = self.useFixture(TempDir()).path
795- destination_dir = os.path.join(path, 'does', 'not', 'exist')
796- task.save_package(path, destination_dir, package_name)
797- self.assertThat(destination_dir, DirExists())
798- self.assertThat(destination_dir, HasPermissions('0755'))
799-
800- def test_save_package_destination_is_file(self):
801- # If the destination directory is actually a file, we raise an error.
802- metadata = self.factory.make_metadata()
803- package_name = metadata['package_name']
804- task = BuildPackageTask()
805- path = self.useFixture(TempDir()).path
806- destination_dir = os.path.join(path, 'some-file')
807- with open(destination_dir, 'w') as fp:
808- fp.write('foo')
809- e = self.assertRaises(
810- OSError,
811- task.save_package, path, destination_dir, package_name)
812- self.assertEqual(errno.EEXIST, e.errno)
813-
814- def test_save_package_destination_exists(self):
815- # If the destination directory already exists, we proceed normally.
816- metadata = self.factory.make_metadata()
817- package_name = metadata['package_name']
818- task = BuildPackageTask()
819- path = self.useFixture(TempDir()).path
820- destination_dir = self.useFixture(TempDir()).path
821- task.save_package(path, destination_dir, package_name)
822- self.assertThat(destination_dir, DirExists())
823-
824- def test_save_package_creates_tarball(self):
825- # save_package creates a tarball in the destination directory that's
826- # named after the package and contains whatever was in the source
827- # directory.
828- metadata = self.factory.make_metadata()
829- package_name = metadata['package_name']
830- task = BuildPackageTask()
831- path = self.useFixture(TempDir()).path
832- os.mkdir(os.path.join(path, 'a-directory'))
833- with open(os.path.join(path, 'a-file'), 'w') as fp:
834- fp.write('foo')
835- destination_dir = self.useFixture(TempDir()).path
836- destination = task.save_package(path, destination_dir, package_name)
837- self.assertEqual(
838- os.path.join(destination_dir, '%s.tar.gz' % (package_name,)),
839- destination)
840- self.assertThat(
841- destination,
842- TarballContains(['a-directory', 'a-file']))
843-
844- def test_save_package_uniquifies(self):
845- # If package-name.tar.gz exists, try package-name-1.tar.gz, etc.
846- metadata = self.factory.make_metadata()
847- package_name = metadata['package_name']
848- task = BuildPackageTask()
849- path = self.useFixture(TempDir()).path
850- os.mkdir(os.path.join(path, 'a-directory'))
851- with open(os.path.join(path, 'a-file'), 'w') as fp:
852- fp.write('foo')
853- destination_dir = self.useFixture(TempDir()).path
854- task.save_package(path, destination_dir, package_name)
855- destination = task.save_package(path, destination_dir, package_name)
856- self.assertEqual(
857- destination,
858- os.path.join(destination_dir, '%s-1.tar.gz' % (package_name,)))
859- destination = task.save_package(path, destination_dir, package_name)
860- self.assertEqual(
861- destination,
862- os.path.join(destination_dir, '%s-2.tar.gz' % (package_name,)))
863-
864 @patch('djpkgme.tasks.submit_error')
865 @patch('djpkgme.tasks.submit_pkgme_info')
866 @patch('djpkgme.tasks.BuildPackageTask.build_package')
867@@ -418,86 +381,6 @@
868 task = BuildPackageTask()
869 self.assertRaises(KeyboardInterrupt, task.run, metadata, {})
870
871- def test_exc_info_as_dict(self):
872- try:
873- raise RuntimeError('whatever')
874- except RuntimeError:
875- exc_info = sys.exc_info()
876- error_dict = tasks.exc_info_as_dict(exc_info)
877- self.assertEqual(
878- {'type': 'RuntimeError',
879- 'traceback': ''.join(traceback.format_exception(*exc_info)),
880- }, error_dict)
881-
882- @patch('djpkgme.tasks.download_file')
883- def test_prepare_icons_downloads_icons(self, mock_download):
884- icons = {
885- '48x48': 'http://whatever/something.png',
886- '64x64': 'http://whatever/foo/another.jpg',
887- }
888- path = self.useFixture(TempDir()).path
889- task = BuildPackageTask()
890- task.prepare_icons(path, icons)
891- self.assertEqual(
892- sorted(
893- [mock.call(
894- 'http://whatever/something.png',
895- name='something.png', working_dir=path),
896- mock.call(
897- 'http://whatever/foo/another.jpg',
898- name='another.jpg', working_dir=path),
899- ]),
900- sorted(mock_download.call_args_list))
901-
902- def test_prepare_icons_returns_paths(self):
903- def download_file(url, name, working_dir):
904- return '%s/++download++/%s' % (working_dir, name)
905- self.patch(tasks, 'download_file', download_file)
906- icons = {
907- '48x48': 'http://whatever/something.png',
908- '64x64': 'http://whatever/foo/another.jpg',
909- }
910- path = self.useFixture(TempDir()).path
911- task = BuildPackageTask()
912- new_icons = task.prepare_icons(path, icons)
913- self.assertEqual(
914- {'48x48': '%s/++download++/something.png' % (path,),
915- '64x64': '%s/++download++/another.jpg' % (path,),
916- }, new_icons)
917-
918- def test_prepare_metadata_default(self):
919- path = self.useFixture(TempDir()).path
920- metadata = self.factory.make_metadata()
921- task = BuildPackageTask()
922- prepared = task.prepare_metadata(path, metadata)
923- self.assertEqual(prepared, metadata)
924-
925- def test_prepare_metadata_maintainer_provided(self):
926- path = self.useFixture(TempDir()).path
927- metadata = self.factory.make_metadata()
928- metadata[MetadataBackend.MAINTAINER] = (
929- 'Upstream <upstream@example.com>')
930- task = BuildPackageTask()
931- prepared = task.prepare_metadata(path, metadata)
932- self.assertEqual(prepared, metadata)
933-
934- @patch('djpkgme.tasks.BuildPackageTask.prepare_icons')
935- def test_prepare_metadata_icons(self, prepare_icons):
936- prepare_icons.return_value = {'48x48': '/tmp/whatever.png'}
937- path = self.useFixture(TempDir()).path
938- metadata = self.factory.make_metadata()
939- expected = dict(metadata)
940- metadata['icons'] = {
941- '48x48': 'http://example.com/whatever.png',
942- }
943- expected[MetadataBackend.ICONS] = {
944- '48x48': '/tmp/whatever.png',
945- }
946- task = BuildPackageTask()
947- prepared = task.prepare_metadata(path, metadata)
948- self.assertEqual(expected, prepared)
949-
950-
951
952 class TestBuildPackageTaskIntegration(TestCaseWithFactory):
953 """Tests that actually build the package."""

Subscribers

People subscribed via source and target branches