Launchpad itself

Merge lp:~danilo/launchpad/bug-494106 into lp:launchpad

  1. bug-494106
  2. Merge into devel
Proposed by Данило Шеган
Status: Merged
Approved by: Данило Шеган
Approved revision: not available
Merged at revision: not available
Proposed branch: lp:~danilo/launchpad/bug-494106
Merge into: lp:launchpad
Diff against target: 155 lines (+69/-17)
3 files modified
lib/lp/translations/model/pofile.py (+22/-8)
lib/lp/translations/scripts/fix_translation_credits.py (+2/-7)
lib/lp/translations/tests/test_pofile.py (+45/-2)
To merge this branch: bzr merge lp:~danilo/launchpad/bug-494106
Reviewer Review Type Date Requested Status
Brad Crittenden (community) code Approve
Review via email: mp+15824@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Данило Шеган (danilo) wrote :

= Bug 494106 =

fix_translation_credits script doesn't pick up where it left off: it simply goes faster through already processed entries, but that means that it runs very, very slow if it runs into locks and such, because it seems DBLoopTuner takes those into account as well.

To fix it, we introduce option to go only through those which have not been fixed yet.

Note: this is a CP candidate.

= Demo & QA =

Query has been QAd on staging, script will be as well.

= Tests =

bin/test -vvct Credits_untranslated

= Launchpad lint =

Checking for conflicts. and issues in doctests and templates.
Running jslint, xmllint, pyflakes, and pylint.
Using normal rules.

Linting changed files:
  lib/lp/translations/model/pofile.py
  lib/lp/translations/scripts/fix_translation_credits.py
  lib/lp/translations/tests/test_pofile.py

Revision history for this message
Brad Crittenden (bac) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/translations/model/pofile.py'
2--- lib/lp/translations/model/pofile.py 2009-11-27 19:25:30 +0000
3+++ lib/lp/translations/model/pofile.py 2009-12-08 17:25:26 +0000
4@@ -70,7 +70,7 @@
5 from lp.translations.utilities.translation_common_format import (
6 TranslationMessageData)
7
8-from storm.expr import And, In, Join, LeftJoin, Or, SQL
9+from storm.expr import And, Exists, In, Join, LeftJoin, Not, Or, Select, SQL
10 from storm.info import ClassAlias
11 from storm.store import Store
12
13@@ -1576,15 +1576,29 @@
14 return POFile.select(
15 "id >= %s" % quote(starting_id), orderBy="id", limit=batch_size)
16
17- def getPOFilesWithTranslationCredits(self):
18+ def getPOFilesWithTranslationCredits(self, untranslated=False):
19 """See `IPOFileSet`."""
20 store = getUtility(IStoreSelector).get(MAIN_STORE, MASTER_FLAVOR)
21- result = store.find(
22- (POFile, POTMsgSet),
23- TranslationTemplateItem.potemplateID == POFile.potemplateID,
24- POTMsgSet.id == TranslationTemplateItem.potmsgsetID,
25- POTMsgSet.msgid_singular == POMsgID.id,
26- In(POMsgID.msgid, POTMsgSet.credits_message_ids))
27+ clauses = [
28+ TranslationTemplateItem.potemplateID == POFile.potemplateID,
29+ POTMsgSet.id == TranslationTemplateItem.potmsgsetID,
30+ POTMsgSet.msgid_singular == POMsgID.id,
31+ In(POMsgID.msgid, POTMsgSet.credits_message_ids)]
32+ if untranslated:
33+ message_select = Select(
34+ True,
35+ And(
36+ TranslationMessage.potmsgsetID == POTMsgSet.id,
37+ TranslationMessage.potemplate == None,
38+ POFile.languageID == TranslationMessage.languageID,
39+ Or(And(
40+ POFile.variant == None,
41+ TranslationMessage.variant == None),
42+ POFile.variant == TranslationMessage.variant),
43+ TranslationMessage.is_current == True),
44+ (TranslationMessage))
45+ clauses.append(Not(Exists(message_select)))
46+ result = store.find((POFile, POTMsgSet), clauses)
47 return result.order_by('POFile.id')
48
49 def getPOFilesTouchedSince(self, date):
50
51=== modified file 'lib/lp/translations/scripts/fix_translation_credits.py'
52--- lib/lp/translations/scripts/fix_translation_credits.py 2009-10-30 15:00:12 +0000
53+++ lib/lp/translations/scripts/fix_translation_credits.py 2009-12-08 17:25:26 +0000
54@@ -9,19 +9,13 @@
55 __all__ = ['VerifyPOFileStatsProcess']
56
57
58-from datetime import datetime, timedelta
59 import logging
60-import pytz
61
62 from zope.component import getUtility
63 from zope.interface import implements
64
65-from canonical.config import config
66-from canonical.launchpad import helpers
67 from canonical.launchpad.interfaces.looptuner import ITunableLoop
68 from lp.translations.interfaces.pofile import IPOFileSet
69-from lp.services.mail.sendmail import simple_sendmail
70-from canonical.launchpad.mailnotification import MailWrapper
71 from canonical.launchpad.utilities.looptuner import DBLoopTuner
72
73
74@@ -35,7 +29,8 @@
75 self.start_at = start_at
76
77 pofileset = getUtility(IPOFileSet)
78- self.pofiles = pofileset.getPOFilesWithTranslationCredits()
79+ self.pofiles = pofileset.getPOFilesWithTranslationCredits(
80+ untranslated=True)
81 self.logger.info(
82 "Figuring out POFiles that need fixing: this may take a while...")
83 self.total = self.pofiles.count()
84
85=== modified file 'lib/lp/translations/tests/test_pofile.py'
86--- lib/lp/translations/tests/test_pofile.py 2009-11-25 15:06:43 +0000
87+++ lib/lp/translations/tests/test_pofile.py 2009-12-08 17:25:26 +0000
88@@ -20,7 +20,7 @@
89 from lp.translations.interfaces.translationcommonformat import (
90 ITranslationFileData)
91 from lp.testing import TestCaseWithFactory, verifyObject
92-from canonical.testing import ZopelessDatabaseLayer
93+from canonical.testing import LaunchpadZopelessLayer, ZopelessDatabaseLayer
94 from canonical.launchpad.webapp.publisher import canonical_url
95
96
97@@ -1210,7 +1210,7 @@
98 class TestPOFileSet(TestCaseWithFactory):
99 """Test PO file set methods."""
100
101- layer = ZopelessDatabaseLayer
102+ layer = LaunchpadZopelessLayer
103
104 def setUp(self):
105 # Create a POFileSet to work with.
106@@ -1447,6 +1447,49 @@
107 list_of_tuples_into_list(
108 self.pofileset.getPOFilesWithTranslationCredits()))
109
110+ def test_getPOFilesWithTranslationCredits_untranslated(self):
111+ # We need absolute DB access to be able to remove a translation
112+ # message.
113+ LaunchpadZopelessLayer.switchDbUser('postgres')
114+
115+ # Initially, we only get data from the sampledata, all of which
116+ # are untranslated.
117+ sampledata_pofiles = list(
118+ self.pofileset.getPOFilesWithTranslationCredits(
119+ untranslated=True))
120+ total = len(sampledata_pofiles)
121+ self.assertEquals(3, total)
122+
123+ # All POFiles with translation credits messages are
124+ # returned along with relevant POTMsgSets.
125+ potemplate1 = self.factory.makePOTemplate()
126+ credits_potmsgset = self.factory.makePOTMsgSet(
127+ potemplate1, singular=u'translator-credits', sequence=1)
128+
129+ sr_pofile = self.factory.makePOFile('sr', potemplate=potemplate1)
130+ pofiles_with_credits = (
131+ self.pofileset.getPOFilesWithTranslationCredits(
132+ untranslated=True))
133+ self.assertNotIn((sr_pofile, credits_potmsgset),
134+ list(pofiles_with_credits))
135+ self.assertEquals(
136+ total,
137+ pofiles_with_credits.count())
138+
139+ # Removing a translation for this message, removes it
140+ # from a result set when untranslated=True is passed in.
141+ message = credits_potmsgset.getSharedTranslationMessage(
142+ sr_pofile.language)
143+ message.destroySelf()
144+ pofiles_with_credits = (
145+ self.pofileset.getPOFilesWithTranslationCredits(
146+ untranslated=True))
147+ self.assertIn((sr_pofile, credits_potmsgset),
148+ list(pofiles_with_credits))
149+ self.assertEquals(
150+ total + 1,
151+ pofiles_with_credits.count())
152+
153
154 class TestPOFileStatistics(TestCaseWithFactory):
155 """Test PO files statistics calculation."""