Launchpad itself

Merge lp:~danilo/launchpad/bug-494106 into lp:launchpad

  1. bug-494106
  2. Merge into devel
Proposed by Данило Шеган
Status: Merged
Approved by: Данило Шеган
Approved revision: not available
Merged at revision: not available
Proposed branch: lp:~danilo/launchpad/bug-494106
Merge into: lp:launchpad
Diff against target: 155 lines (+69/-17)
3 files modified
lib/lp/translations/model/pofile.py (+22/-8)
lib/lp/translations/scripts/fix_translation_credits.py (+2/-7)
lib/lp/translations/tests/test_pofile.py (+45/-2)
To merge this branch: bzr merge lp:~danilo/launchpad/bug-494106
Reviewer Review Type Date Requested Status
Brad Crittenden (community) code Approve
Review via email: mp+15824@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Данило Шеган (danilo) wrote :

= Bug 494106 =

fix_translation_credits script doesn't pick up where it left off: it simply goes faster through already processed entries, but that means that it runs very, very slow if it runs into locks and such, because it seems DBLoopTuner takes those into account as well.

To fix it, we introduce option to go only through those which have not been fixed yet.

Note: this is a CP candidate.

= Demo & QA =

Query has been QAd on staging, script will be as well.

= Tests =

bin/test -vvct Credits_untranslated

= Launchpad lint =

Checking for conflicts. and issues in doctests and templates.
Running jslint, xmllint, pyflakes, and pylint.
Using normal rules.

Linting changed files:
  lib/lp/translations/model/pofile.py
  lib/lp/translations/scripts/fix_translation_credits.py
  lib/lp/translations/tests/test_pofile.py

Revision history for this message
Brad Crittenden (bac) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/translations/model/pofile.py'
--- lib/lp/translations/model/pofile.py 2009-11-27 19:25:30 +0000
+++ lib/lp/translations/model/pofile.py 2009-12-08 17:25:26 +0000
@@ -70,7 +70,7 @@
70from lp.translations.utilities.translation_common_format import (70from lp.translations.utilities.translation_common_format import (
71 TranslationMessageData)71 TranslationMessageData)
7272
73from storm.expr import And, In, Join, LeftJoin, Or, SQL73from storm.expr import And, Exists, In, Join, LeftJoin, Not, Or, Select, SQL
74from storm.info import ClassAlias74from storm.info import ClassAlias
75from storm.store import Store75from storm.store import Store
7676
@@ -1576,15 +1576,29 @@
1576 return POFile.select(1576 return POFile.select(
1577 "id >= %s" % quote(starting_id), orderBy="id", limit=batch_size)1577 "id >= %s" % quote(starting_id), orderBy="id", limit=batch_size)
15781578
1579 def getPOFilesWithTranslationCredits(self):1579 def getPOFilesWithTranslationCredits(self, untranslated=False):
1580 """See `IPOFileSet`."""1580 """See `IPOFileSet`."""
1581 store = getUtility(IStoreSelector).get(MAIN_STORE, MASTER_FLAVOR)1581 store = getUtility(IStoreSelector).get(MAIN_STORE, MASTER_FLAVOR)
1582 result = store.find(1582 clauses = [
1583 (POFile, POTMsgSet),1583 TranslationTemplateItem.potemplateID == POFile.potemplateID,
1584 TranslationTemplateItem.potemplateID == POFile.potemplateID,1584 POTMsgSet.id == TranslationTemplateItem.potmsgsetID,
1585 POTMsgSet.id == TranslationTemplateItem.potmsgsetID,1585 POTMsgSet.msgid_singular == POMsgID.id,
1586 POTMsgSet.msgid_singular == POMsgID.id,1586 In(POMsgID.msgid, POTMsgSet.credits_message_ids)]
1587 In(POMsgID.msgid, POTMsgSet.credits_message_ids))1587 if untranslated:
1588 message_select = Select(
1589 True,
1590 And(
1591 TranslationMessage.potmsgsetID == POTMsgSet.id,
1592 TranslationMessage.potemplate == None,
1593 POFile.languageID == TranslationMessage.languageID,
1594 Or(And(
1595 POFile.variant == None,
1596 TranslationMessage.variant == None),
1597 POFile.variant == TranslationMessage.variant),
1598 TranslationMessage.is_current == True),
1599 (TranslationMessage))
1600 clauses.append(Not(Exists(message_select)))
1601 result = store.find((POFile, POTMsgSet), clauses)
1588 return result.order_by('POFile.id')1602 return result.order_by('POFile.id')
15891603
1590 def getPOFilesTouchedSince(self, date):1604 def getPOFilesTouchedSince(self, date):
15911605
=== modified file 'lib/lp/translations/scripts/fix_translation_credits.py'
--- lib/lp/translations/scripts/fix_translation_credits.py 2009-10-30 15:00:12 +0000
+++ lib/lp/translations/scripts/fix_translation_credits.py 2009-12-08 17:25:26 +0000
@@ -9,19 +9,13 @@
9__all__ = ['VerifyPOFileStatsProcess']9__all__ = ['VerifyPOFileStatsProcess']
1010
1111
12from datetime import datetime, timedelta
13import logging12import logging
14import pytz
1513
16from zope.component import getUtility14from zope.component import getUtility
17from zope.interface import implements15from zope.interface import implements
1816
19from canonical.config import config
20from canonical.launchpad import helpers
21from canonical.launchpad.interfaces.looptuner import ITunableLoop17from canonical.launchpad.interfaces.looptuner import ITunableLoop
22from lp.translations.interfaces.pofile import IPOFileSet18from lp.translations.interfaces.pofile import IPOFileSet
23from lp.services.mail.sendmail import simple_sendmail
24from canonical.launchpad.mailnotification import MailWrapper
25from canonical.launchpad.utilities.looptuner import DBLoopTuner19from canonical.launchpad.utilities.looptuner import DBLoopTuner
2620
2721
@@ -35,7 +29,8 @@
35 self.start_at = start_at29 self.start_at = start_at
3630
37 pofileset = getUtility(IPOFileSet)31 pofileset = getUtility(IPOFileSet)
38 self.pofiles = pofileset.getPOFilesWithTranslationCredits()32 self.pofiles = pofileset.getPOFilesWithTranslationCredits(
33 untranslated=True)
39 self.logger.info(34 self.logger.info(
40 "Figuring out POFiles that need fixing: this may take a while...")35 "Figuring out POFiles that need fixing: this may take a while...")
41 self.total = self.pofiles.count()36 self.total = self.pofiles.count()
4237
=== modified file 'lib/lp/translations/tests/test_pofile.py'
--- lib/lp/translations/tests/test_pofile.py 2009-11-25 15:06:43 +0000
+++ lib/lp/translations/tests/test_pofile.py 2009-12-08 17:25:26 +0000
@@ -20,7 +20,7 @@
20from lp.translations.interfaces.translationcommonformat import (20from lp.translations.interfaces.translationcommonformat import (
21 ITranslationFileData)21 ITranslationFileData)
22from lp.testing import TestCaseWithFactory, verifyObject22from lp.testing import TestCaseWithFactory, verifyObject
23from canonical.testing import ZopelessDatabaseLayer23from canonical.testing import LaunchpadZopelessLayer, ZopelessDatabaseLayer
24from canonical.launchpad.webapp.publisher import canonical_url24from canonical.launchpad.webapp.publisher import canonical_url
2525
2626
@@ -1210,7 +1210,7 @@
1210class TestPOFileSet(TestCaseWithFactory):1210class TestPOFileSet(TestCaseWithFactory):
1211 """Test PO file set methods."""1211 """Test PO file set methods."""
12121212
1213 layer = ZopelessDatabaseLayer1213 layer = LaunchpadZopelessLayer
12141214
1215 def setUp(self):1215 def setUp(self):
1216 # Create a POFileSet to work with.1216 # Create a POFileSet to work with.
@@ -1447,6 +1447,49 @@
1447 list_of_tuples_into_list(1447 list_of_tuples_into_list(
1448 self.pofileset.getPOFilesWithTranslationCredits()))1448 self.pofileset.getPOFilesWithTranslationCredits()))
14491449
1450 def test_getPOFilesWithTranslationCredits_untranslated(self):
1451 # We need absolute DB access to be able to remove a translation
1452 # message.
1453 LaunchpadZopelessLayer.switchDbUser('postgres')
1454
1455 # Initially, we only get data from the sampledata, all of which
1456 # are untranslated.
1457 sampledata_pofiles = list(
1458 self.pofileset.getPOFilesWithTranslationCredits(
1459 untranslated=True))
1460 total = len(sampledata_pofiles)
1461 self.assertEquals(3, total)
1462
1463 # All POFiles with translation credits messages are
1464 # returned along with relevant POTMsgSets.
1465 potemplate1 = self.factory.makePOTemplate()
1466 credits_potmsgset = self.factory.makePOTMsgSet(
1467 potemplate1, singular=u'translator-credits', sequence=1)
1468
1469 sr_pofile = self.factory.makePOFile('sr', potemplate=potemplate1)
1470 pofiles_with_credits = (
1471 self.pofileset.getPOFilesWithTranslationCredits(
1472 untranslated=True))
1473 self.assertNotIn((sr_pofile, credits_potmsgset),
1474 list(pofiles_with_credits))
1475 self.assertEquals(
1476 total,
1477 pofiles_with_credits.count())
1478
1479 # Removing a translation for this message, removes it
1480 # from a result set when untranslated=True is passed in.
1481 message = credits_potmsgset.getSharedTranslationMessage(
1482 sr_pofile.language)
1483 message.destroySelf()
1484 pofiles_with_credits = (
1485 self.pofileset.getPOFilesWithTranslationCredits(
1486 untranslated=True))
1487 self.assertIn((sr_pofile, credits_potmsgset),
1488 list(pofiles_with_credits))
1489 self.assertEquals(
1490 total + 1,
1491 pofiles_with_credits.count())
1492
14501493
1451class TestPOFileStatistics(TestCaseWithFactory):1494class TestPOFileStatistics(TestCaseWithFactory):
1452 """Test PO files statistics calculation."""1495 """Test PO files statistics calculation."""