Launchpad itself

Merge lp:~jelmer/launchpad/55288-publisher-unknown-distroseries into lp:launchpad

  1. 55288-publisher-unknown-distroseries
  2. Merge into devel
Proposed by Jelmer Vernooij
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11323
Proposed branch: lp:~jelmer/launchpad/55288-publisher-unknown-distroseries
Merge into: lp:launchpad
Diff against target: 236 lines (+67/-28)
4 files modified
lib/lp/archivepublisher/config.py (+19/-14)
lib/lp/archivepublisher/ftparchive.py (+11/-3)
lib/lp/archivepublisher/tests/test_config.py (+19/-9)
lib/lp/archivepublisher/tests/test_ftparchive.py (+18/-2)
To merge this branch: bzr merge lp:~jelmer/launchpad/55288-publisher-unknown-distroseries
Reviewer Review Type Date Requested Status
Leonard Richardson (community) code Approve
Review via email: mp+32232@code.launchpad.net

Commit message

Warn about distroseries without lucilleconfig rather than failing the entire publisher run.

Description of the change

This makes the publisher more robust against distroseries that have not been initialized for soyuz yet.

Rather than having the publisher configuration raise a LucilleConfigError when a distroseries doesn't have a matching entry in the lucille configuration, it will now write a warning to the log file.

To post a comment you must log in.
Revision history for this message
Leonard Richardson (leonardr) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/archivepublisher/config.py'
--- lib/lp/archivepublisher/config.py 2010-07-07 06:28:03 +0000
+++ lib/lp/archivepublisher/config.py 2010-08-10 17:43:45 +0000
@@ -120,18 +120,15 @@
120 config_segment["archtags"].append(120 config_segment["archtags"].append(
121 dar.architecturetag.encode('utf-8'))121 dar.architecturetag.encode('utf-8'))
122122
123 if not dr.lucilleconfig:123 if dr.lucilleconfig:
124 raise LucilleConfigError(124 strio = StringIO(dr.lucilleconfig.encode('utf-8'))
125 'No Lucille configuration section for %s' % dr.name)125 config_segment["config"] = ConfigParser()
126126 config_segment["config"].readfp(strio)
127 strio = StringIO(dr.lucilleconfig.encode('utf-8'))127 strio.close()
128 config_segment["config"] = ConfigParser()128 config_segment["components"] = config_segment["config"].get(
129 config_segment["config"].readfp(strio)129 "publishing", "components").split(" ")
130 strio.close()130
131 config_segment["components"] = config_segment["config"].get(131 self._distroseries[distroseries_name] = config_segment
132 "publishing", "components").split(" ")
133
134 self._distroseries[distroseries_name] = config_segment
135132
136 strio = StringIO(distribution.lucilleconfig.encode('utf-8'))133 strio = StringIO(distribution.lucilleconfig.encode('utf-8'))
137 self._distroconfig = ConfigParser()134 self._distroconfig = ConfigParser()
@@ -144,11 +141,19 @@
144 # Because dicts iterate for keys only; this works to get dr names141 # Because dicts iterate for keys only; this works to get dr names
145 return self._distroseries.keys()142 return self._distroseries.keys()
146143
144 def series(self, dr):
145 try:
146 return self._distroseries[dr]
147 except KeyError:
148 raise LucilleConfigError(
149 'No Lucille config section for %s in %s' %
150 (dr, self.distroName))
151
147 def archTagsForSeries(self, dr):152 def archTagsForSeries(self, dr):
148 return self._distroseries[dr]["archtags"]153 return self.series(dr)["archtags"]
149154
150 def componentsForSeries(self, dr):155 def componentsForSeries(self, dr):
151 return self._distroseries[dr]["components"]156 return self.series(dr)["components"]
152157
153 def _extractConfigInfo(self):158 def _extractConfigInfo(self):
154 """Extract configuration information into the attributes we use"""159 """Extract configuration information into the attributes we use"""
155160
=== modified file 'lib/lp/archivepublisher/ftparchive.py'
--- lib/lp/archivepublisher/ftparchive.py 2009-10-26 18:40:04 +0000
+++ lib/lp/archivepublisher/ftparchive.py 2010-08-10 17:43:45 +0000
@@ -138,6 +138,14 @@
138 self._config = config138 self._config = config
139 self._diskpool = diskpool139 self._diskpool = diskpool
140 self.distro = distro140 self.distro = distro
141 self.distroseries = []
142 for distroseries in self.distro.series:
143 if not distroseries.name in self._config.distroSeriesNames():
144 self.log.warning("Distroseries %s in %s doesn't have "
145 "a lucille configuration.", distroseries.name,
146 self.distro.name)
147 else:
148 self.distroseries.append(distroseries)
141 self.publisher = publisher149 self.publisher = publisher
142 self.release_files_needed = {}150 self.release_files_needed = {}
143151
@@ -185,7 +193,7 @@
185 # iterate over the pockets, and do the suffix check inside193 # iterate over the pockets, and do the suffix check inside
186 # createEmptyPocketRequest; that would also allow us to replace194 # createEmptyPocketRequest; that would also allow us to replace
187 # the == "" check we do there by a RELEASE match195 # the == "" check we do there by a RELEASE match
188 for distroseries in self.distro:196 for distroseries in self.distroseries:
189 components = self._config.componentsForSeries(distroseries.name)197 components = self._config.componentsForSeries(distroseries.name)
190 for pocket, suffix in pocketsuffix.items():198 for pocket, suffix in pocketsuffix.items():
191 if not fullpublish:199 if not fullpublish:
@@ -366,7 +374,7 @@
366374
367 def generateOverrides(self, fullpublish=False):375 def generateOverrides(self, fullpublish=False):
368 """Collect packages that need overrides, and generate them."""376 """Collect packages that need overrides, and generate them."""
369 for distroseries in self.distro.series:377 for distroseries in self.distroseries:
370 for pocket in PackagePublishingPocket.items:378 for pocket in PackagePublishingPocket.items:
371 if not fullpublish:379 if not fullpublish:
372 if not self.publisher.isDirty(distroseries, pocket):380 if not self.publisher.isDirty(distroseries, pocket):
@@ -629,7 +637,7 @@
629637
630 def generateFileLists(self, fullpublish=False):638 def generateFileLists(self, fullpublish=False):
631 """Collect currently published FilePublishings and write filelists."""639 """Collect currently published FilePublishings and write filelists."""
632 for distroseries in self.distro.series:640 for distroseries in self.distroseries:
633 for pocket in pocketsuffix:641 for pocket in pocketsuffix:
634 if not fullpublish:642 if not fullpublish:
635 if not self.publisher.isDirty(distroseries, pocket):643 if not self.publisher.isDirty(distroseries, pocket):
636644
=== modified file 'lib/lp/archivepublisher/tests/test_config.py'
--- lib/lp/archivepublisher/tests/test_config.py 2010-07-18 00:24:06 +0000
+++ lib/lp/archivepublisher/tests/test_config.py 2010-08-10 17:43:45 +0000
@@ -5,36 +5,48 @@
55
6__metaclass__ = type6__metaclass__ = type
77
8import unittest
9
10from zope.component import getUtility8from zope.component import getUtility
119
12from canonical.config import config10from canonical.config import config
13from canonical.launchpad.interfaces import IDistributionSet11from canonical.launchpad.interfaces import IDistributionSet
14from canonical.testing import LaunchpadZopelessLayer12from canonical.testing import LaunchpadZopelessLayer
1513
1614from lp.archivepublisher.config import Config, LucilleConfigError
17class TestConfig(unittest.TestCase):15from lp.testing import TestCaseWithFactory
16
17
18class TestConfig(TestCaseWithFactory):
18 layer = LaunchpadZopelessLayer19 layer = LaunchpadZopelessLayer
1920
20 def setUp(self):21 def setUp(self):
22 super(TestConfig, self).setUp()
21 self.layer.switchDbUser(config.archivepublisher.dbuser)23 self.layer.switchDbUser(config.archivepublisher.dbuser)
22 self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']24 self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']
2325
26 def testMissingDistroSeries(self):
27 distroseries = self.factory.makeDistroSeries(
28 distribution=self.ubuntutest, name="somename")
29 d = Config(self.ubuntutest)
30 dsns = d.distroSeriesNames()
31 self.assertEquals(len(dsns), 2)
32 self.assertEquals(dsns[0], "breezy-autotest")
33 self.assertEquals(dsns[1], "hoary-test")
34 self.assertRaises(LucilleConfigError,
35 d.archTagsForSeries, "somename")
36 self.assertRaises(LucilleConfigError,
37 d.archTagsForSeries, "unknown")
38
24 def testInstantiate(self):39 def testInstantiate(self):
25 """Config should instantiate"""40 """Config should instantiate"""
26 from lp.archivepublisher.config import Config
27 d = Config(self.ubuntutest)41 d = Config(self.ubuntutest)
2842
29 def testDistroName(self):43 def testDistroName(self):
30 """Config should be able to return the distroName"""44 """Config should be able to return the distroName"""
31 from lp.archivepublisher.config import Config
32 d = Config(self.ubuntutest)45 d = Config(self.ubuntutest)
33 self.assertEqual(d.distroName, "ubuntutest")46 self.assertEqual(d.distroName, "ubuntutest")
3447
35 def testDistroSeriesNames(self):48 def testDistroSeriesNames(self):
36 """Config should return two distroseries names"""49 """Config should return two distroseries names"""
37 from lp.archivepublisher.config import Config
38 d = Config(self.ubuntutest)50 d = Config(self.ubuntutest)
39 dsns = d.distroSeriesNames()51 dsns = d.distroSeriesNames()
40 self.assertEquals(len(dsns), 2)52 self.assertEquals(len(dsns), 2)
@@ -43,14 +55,12 @@
4355
44 def testArchTagsForSeries(self):56 def testArchTagsForSeries(self):
45 """Config should have the arch tags for the drs"""57 """Config should have the arch tags for the drs"""
46 from lp.archivepublisher.config import Config
47 d = Config(self.ubuntutest)58 d = Config(self.ubuntutest)
48 archs = d.archTagsForSeries("hoary-test")59 archs = d.archTagsForSeries("hoary-test")
49 self.assertEquals(len(archs), 2)60 self.assertEquals(len(archs), 2)
5061
51 def testDistroConfig(self):62 def testDistroConfig(self):
52 """Config should have parsed a distro config"""63 """Config should have parsed a distro config"""
53 from lp.archivepublisher.config import Config
54 d = Config(self.ubuntutest)64 d = Config(self.ubuntutest)
55 # NOTE: Add checks here when you add stuff in util.py65 # NOTE: Add checks here when you add stuff in util.py
56 self.assertEquals(d.stayofexecution, 5)66 self.assertEquals(d.stayofexecution, 5)
5767
=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
--- lib/lp/archivepublisher/tests/test_ftparchive.py 2010-07-18 00:24:06 +0000
+++ lib/lp/archivepublisher/tests/test_ftparchive.py 2010-08-10 17:43:45 +0000
@@ -15,7 +15,7 @@
15from zope.component import getUtility15from zope.component import getUtility
1616
17from canonical.config import config17from canonical.config import config
18from canonical.launchpad.scripts.logger import QuietFakeLogger18from canonical.launchpad.scripts.logger import BufferLogger, QuietFakeLogger
19from canonical.testing import LaunchpadZopelessLayer19from canonical.testing import LaunchpadZopelessLayer
20from lp.archivepublisher.config import Config20from lp.archivepublisher.config import Config
21from lp.archivepublisher.diskpool import DiskPool21from lp.archivepublisher.diskpool import DiskPool
@@ -23,6 +23,7 @@
23from lp.archivepublisher.publishing import Publisher23from lp.archivepublisher.publishing import Publisher
24from lp.registry.interfaces.distribution import IDistributionSet24from lp.registry.interfaces.distribution import IDistributionSet
25from lp.registry.interfaces.pocket import PackagePublishingPocket25from lp.registry.interfaces.pocket import PackagePublishingPocket
26from lp.testing import TestCaseWithFactory
2627
2728
28def sanitize_apt_ftparchive_Sources_output(text):29def sanitize_apt_ftparchive_Sources_output(text):
@@ -55,10 +56,11 @@
55 return self._result[i:j]56 return self._result[i:j]
5657
5758
58class TestFTPArchive(unittest.TestCase):59class TestFTPArchive(TestCaseWithFactory):
59 layer = LaunchpadZopelessLayer60 layer = LaunchpadZopelessLayer
6061
61 def setUp(self):62 def setUp(self):
63 super(TestFTPArchive, self).setUp()
62 self.layer.switchDbUser(config.archivepublisher.dbuser)64 self.layer.switchDbUser(config.archivepublisher.dbuser)
6365
64 self._distribution = getUtility(IDistributionSet)['ubuntutest']66 self._distribution = getUtility(IDistributionSet)['ubuntutest']
@@ -79,6 +81,7 @@
79 self._publisher = SamplePublisher(self._archive)81 self._publisher = SamplePublisher(self._archive)
8082
81 def tearDown(self):83 def tearDown(self):
84 super(TestFTPArchive, self).tearDown()
82 shutil.rmtree(self._config.distroroot)85 shutil.rmtree(self._config.distroroot)
8386
84 def _verifyFile(self, filename, directory, output_filter=None):87 def _verifyFile(self, filename, directory, output_filter=None):
@@ -116,6 +119,19 @@
116 self._publisher)119 self._publisher)
117 return fa120 return fa
118121
122 def test_NoLucilleConfig(self):
123 # Distroseries without a lucille configuration get ignored
124 # and trigger a warning, they don't break the publisher
125 logger = BufferLogger()
126 publisher = Publisher(
127 logger, self._config, self._dp, self._archive)
128 self.factory.makeDistroSeries(self._distribution, name="somename")
129 fa = FTPArchiveHandler(logger, self._config, self._dp,
130 self._distribution, publisher)
131 fa.createEmptyPocketRequests(fullpublish=True)
132 self.assertEquals("WARNING: Distroseries somename in ubuntutest doesn't "
133 "have a lucille configuration.\n", logger.buffer.getvalue())
134
119 def test_getSourcesForOverrides(self):135 def test_getSourcesForOverrides(self):
120 # getSourcesForOverrides returns a list of tuples containing:136 # getSourcesForOverrides returns a list of tuples containing:
121 # (sourcename, suite, component, section)137 # (sourcename, suite, component, section)