Launchpad itself

Merge lp:~jelmer/launchpad/publisher-use-debian-1 into lp:launchpad

  1. publisher-use-debian-1
  2. Merge into devel
Proposed by Jelmer Vernooij
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11404
Proposed branch: lp:~jelmer/launchpad/publisher-use-debian-1
Merge into: lp:launchpad
Diff against target: 211 lines (+62/-68)
2 files modified
lib/lp/archivepublisher/publishing.py (+61/-67)
lib/lp/soyuz/doc/soyuz-upload.txt (+1/-1)
To merge this branch: bzr merge lp:~jelmer/launchpad/publisher-use-debian-1
Reviewer Review Type Date Requested Status
Brad Crittenden (community) code Approve
Review via email: mp+32245@code.launchpad.net

Commit message

Use python-debian to generate Release files in the archivepublisher.

Description of the change

Refactor lp.archivepublisher.publishing to use the functionality in the new "debian" Python module to generate Release files, rather than using hardcoded strings.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) wrote :

I haven't done a pre-implementation call about this specific change, although I have discussed moving towards more use of python-debian (which is an upstream project maintained by various Debian and Ubuntu developers) with Julian in the past.

I'm about to make some more changes to this particular part of the code, and it seemed like a good idea to refactor first and use python-debian's infrastructure so that I don't have to worry about the serialization of newly introduced fields.

Tests: ./bin/test lp.archivepublisher

I've fixed all remaining lint issues.

Revision history for this message
Brad Crittenden (bac) wrote :

Very nice branch Jelmer. Please use a more descriptive loop variable than "f", especially since it is used again a little later as a file descriptor. Recall our coding guidelines strongly discourage single letter variables, even in loops.

review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/archivepublisher/publishing.py'
--- lib/lp/archivepublisher/publishing.py 2010-07-07 06:28:03 +0000
+++ lib/lp/archivepublisher/publishing.py 2010-08-20 11:35:17 +0000
@@ -14,6 +14,8 @@
14import os14import os
15import shutil15import shutil
1616
17from debian.deb822 import Release
18
17from datetime import datetime19from datetime import datetime
1820
19from zope.component import getUtility21from zope.component import getUtility
@@ -40,24 +42,6 @@
4042
41suffixpocket = dict((v, k) for (k, v) in pocketsuffix.items())43suffixpocket = dict((v, k) for (k, v) in pocketsuffix.items())
4244
43DISTRORELEASE_STANZA = """Origin: %s
44Label: %s
45Suite: %s
46Version: %s
47Codename: %s
48Date: %s
49Architectures: %s
50Components: %s
51Description: %s
52"""
53
54DISTROARCHRELEASE_STANZA = """Archive: %s
55Version: %s
56Component: %s
57Origin: %s
58Label: %s
59Architecture: %s
60"""
6145
62def reorder_components(components):46def reorder_components(components):
63 """Return a list of the components provided.47 """Return a list of the components provided.
@@ -92,6 +76,7 @@
9276
93 return dp77 return dp
9478
79
95def getPublisher(archive, allowed_suites, log, distsroot=None):80def getPublisher(archive, allowed_suites, log, distsroot=None):
96 """Return an initialised Publisher instance for the given context.81 """Return an initialised Publisher instance for the given context.
9782
@@ -487,33 +472,43 @@
487 else:472 else:
488 drsummary += pocket.name.capitalize()473 drsummary += pocket.name.capitalize()
489474
475 release_file = Release()
476 release_file["Origin"] = self._getOrigin()
477 release_file["Label"] = self._getLabel()
478 release_file["Suite"] = full_name
479 release_file["Version"] = distroseries.version
480 release_file["Codename"] = distroseries.name
481 release_file["Date"] = datetime.utcnow().strftime(
482 "%a, %d %b %Y %k:%M:%S UTC")
483 release_file["Architectures"] = " ".join(
484 sorted(list(all_architectures)))
485 release_file["Components"] = " ".join(
486 reorder_components(all_components))
487 release_file["Description"] = drsummary
488
489 for filename in sorted(list(all_files), key=os.path.dirname):
490 entry = self._readIndexFileContents(full_name, filename)
491 if entry is None:
492 continue
493 release_file.setdefault("MD5Sum", []).append({
494 "md5sum": hashlib.md5(entry).hexdigest(),
495 "name": filename,
496 "size": len(entry)})
497 release_file.setdefault("SHA1", []).append({
498 "sha1": hashlib.sha1(entry).hexdigest(),
499 "name": filename,
500 "size": len(entry)})
501 release_file.setdefault("SHA256", []).append({
502 "sha256": hashlib.sha256(entry).hexdigest(),
503 "name": filename,
504 "size": len(entry)})
505
490 f = open(os.path.join(506 f = open(os.path.join(
491 self._config.distsroot, full_name, "Release"), "w")507 self._config.distsroot, full_name, "Release"), "w")
492508 try:
493 stanza = (DISTRORELEASE_STANZA % (509 release_file.dump(f, "utf-8")
494 self._getOrigin(),510 finally:
495 self._getLabel(),511 f.close()
496 full_name,
497 distroseries.version,
498 distroseries.name,
499 datetime.utcnow().strftime("%a, %d %b %Y %k:%M:%S UTC"),
500 " ".join(sorted(list(all_architectures))),
501 " ".join(reorder_components(all_components)),
502 drsummary)).encode("utf-8")
503 f.write(stanza)
504
505 f.write("MD5Sum:\n")
506 all_files = sorted(list(all_files), key=os.path.dirname)
507 for file_name in all_files:
508 self._writeSumLine(full_name, f, file_name, hashlib.md5)
509 f.write("SHA1:\n")
510 for file_name in all_files:
511 self._writeSumLine(full_name, f, file_name, hashlib.sha1)
512 f.write("SHA256:\n")
513 for file_name in all_files:
514 self._writeSumLine(full_name, f, file_name, hashlib.sha256)
515
516 f.close()
517512
518 # Skip signature if the archive signing key is undefined.513 # Skip signature if the archive signing key is undefined.
519 if self.archive.signing_key is None:514 if self.archive.signing_key is None:
@@ -562,26 +557,29 @@
562557
563 all_files.add(os.path.join(component, architecture, "Release"))558 all_files.add(os.path.join(component, architecture, "Release"))
564559
560 release_file = Release()
561 release_file["Archive"] = full_name
562 release_file["Version"] = distroseries.version
563 release_file["Component"] = component
564 release_file["Origin"] = self._getOrigin()
565 release_file["Label"] = self._getLabel()
566 release_file["Architecture"] = clean_architecture
567
565 f = open(os.path.join(self._config.distsroot, full_name,568 f = open(os.path.join(self._config.distsroot, full_name,
566 component, architecture, "Release"), "w")569 component, architecture, "Release"), "w")
567570 try:
568 stanza = (DISTROARCHRELEASE_STANZA % (571 release_file.dump(f, "utf-8")
569 full_name,572 finally:
570 distroseries.version,573 f.close()
571 component,
572 self._getOrigin(),
573 self._getLabel(),
574 unicode(clean_architecture))).encode("utf-8")
575 f.write(stanza)
576 f.close()
577574
578 return clean_architecture575 return clean_architecture
579576
580 def _writeSumLine(self, distroseries_name, out_file, file_name, sum_form):577 def _readIndexFileContents(self, distroseries_name, file_name):
581 """Write out a checksum line.578 """Read an index files' contents.
582579
583 Writes a checksum to the given file for the given filename in580 :param distroseries_name: Distro series name
584 the given form.581 :param file_name: Filename relative to the parent container directory.
582 :return: File contents, or None if the file could not be found.
585 """583 """
586 full_name = os.path.join(self._config.distsroot,584 full_name = os.path.join(self._config.distsroot,
587 distroseries_name, file_name)585 distroseries_name, file_name)
@@ -590,25 +588,21 @@
590 # Most likely we have an incomplete archive (E.g. no sources588 # Most likely we have an incomplete archive (E.g. no sources
591 # for a given distroseries). This is a non-fatal issue589 # for a given distroseries). This is a non-fatal issue
592 self.log.debug("Failed to find " + full_name)590 self.log.debug("Failed to find " + full_name)
593 return591 return None
594592
595 in_file = open(full_name, 'r')593 in_file = open(full_name, 'r')
596 try:594 try:
597 contents = in_file.read()595 return in_file.read()
598 length = len(contents)
599 checksum = sum_form(contents).hexdigest()
600 finally:596 finally:
601 in_file.close()597 in_file.close()
602598
603 out_file.write(" %s % 16d %s\n" % (checksum, length, file_name))
604
605 def deleteArchive(self):599 def deleteArchive(self):
606 """Delete the archive.600 """Delete the archive.
607 601
608 Physically remove the entire archive from disk and set the archive's 602 Physically remove the entire archive from disk and set the archive's
609 status to DELETED.603 status to DELETED.
610604
611 Any errors encountered while removing the archive from disk will 605 Any errors encountered while removing the archive from disk will
612 be caught and an OOPS report generated.606 be caught and an OOPS report generated.
613 """607 """
614608
@@ -627,7 +621,7 @@
627 self.log.warning(621 self.log.warning(
628 "Failed to delete directory '%s' for archive "622 "Failed to delete directory '%s' for archive "
629 "'%s/%s'\n%s" % (623 "'%s/%s'\n%s" % (
630 directory, self.archive.owner.name, 624 directory, self.archive.owner.name,
631 self.archive.name, e))625 self.archive.name, e))
632626
633 self.archive.status = ArchiveStatus.DELETED627 self.archive.status = ArchiveStatus.DELETED
634628
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt 2010-06-29 14:01:01 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt 2010-08-20 11:35:17 +0000
@@ -708,7 +708,7 @@
708 Version: 6.6.6708 Version: 6.6.6
709 Codename: breezy-autotest709 Codename: breezy-autotest
710 Date: ...710 Date: ...
711 Architectures: 711 Architectures:
712 Components: main restricted universe multiverse712 Components: main restricted universe multiverse
713 Description: ubuntutest Breezy Badger Autotest 6.6.6713 Description: ubuntutest Breezy Badger Autotest 6.6.6
714 MD5Sum:714 MD5Sum: