libdep-service

Merge lp:~jml/libdep-service/copy-in-database into lp:libdep-service

  1. copy-in-database
  2. Merge into trunk
Proposed by Jonathan Lange
Status: Merged
Merged at revision: 80
Proposed branch: lp:~jml/libdep-service/copy-in-database
Merge into: lp:libdep-service
Diff against target: 628 lines (+500/-11)
12 files modified
djlibdep/api.py (+1/-1)
djlibdep/configuration.py (+108/-0)
djlibdep/database.py (+168/-0)
djlibdep/db/patch-00001.sql (+3/-0)
djlibdep/db/patch-00002.sql (+11/-0)
djlibdep/db/postgres_schema.sql (+7/-0)
djlibdep/tasks.py (+1/-1)
djlibdep/testing.py (+192/-0)
djlibdep/tests/helpers.py (+2/-2)
djlibdep/tests/test_interface.py (+5/-5)
djlibdep/tests/test_tasks.py (+1/-1)
djlibdep/views.py (+1/-1)
To merge this branch: bzr merge lp:~jml/libdep-service/copy-in-database
Reviewer Review Type Date Requested Status
James Westby (community) Approve
Review via email: mp+133962@code.launchpad.net

Commit message

Copy over parts of pkgme-devportal that we use

Description of the change

Pretty straightforward.

To post a comment you must log in.
Revision history for this message
James Westby (james-w) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'djlibdep/api.py'
2--- djlibdep/api.py 2012-10-30 13:59:34 +0000
3+++ djlibdep/api.py 2012-11-12 17:37:20 +0000
4@@ -14,7 +14,7 @@
5
6 from .stats import get_metrics
7
8-from devportalbinary.database import get_dependency_database
9+from .database import get_dependency_database
10
11
12 def get_binaries_for_libraries(db, libs, arches):
13
14=== added file 'djlibdep/configuration.py'
15--- djlibdep/configuration.py 1970-01-01 00:00:00 +0000
16+++ djlibdep/configuration.py 2012-11-12 17:37:20 +0000
17@@ -0,0 +1,108 @@
18+import os
19+
20+from configglue import parser
21+
22+from configglue.schema import (
23+ Schema,
24+ Section,
25+ DictOption,
26+ IntOption,
27+ StringOption,
28+ TupleOption,
29+)
30+
31+# The environment variable that controls the config file location.
32+CONF_FILE_ENV_VAR = 'PKGME_DEVPORTAL_CONFIG_FILE'
33+
34+# Where to look if the environment variable isn't set.
35+# XXX: 'pkgme-binary' is the historic name of this package. Change this
36+# to look first in ~/.config/pkgme-devportal/conf and then fall back to
37+# this one. Once production systems are updated to the new config, remove
38+# the fallback.
39+_DEFAULT_CONF_FILE = '~/.config/pkgme-binary/conf'
40+
41+
42+class DevportalSchema(Schema):
43+
44+ # database
45+ database = Section()
46+ database.db_type = StringOption(
47+ default='aptfile',
48+ help=('The database to use, "postgres", "aptfile" are supported '
49+ 'values.'))
50+ database.host = StringOption(
51+ default=None,
52+ help='The database host (for postgres)')
53+ database.port = IntOption(
54+ default=None,
55+ help='The database port (for postgres)')
56+ database.username = StringOption(
57+ default=None,
58+ help='The database username (for postgres)')
59+ database.password = StringOption(
60+ default=None,
61+ help='The database password (for postgres)')
62+ database.db_name = StringOption(
63+ default=None,
64+ help='The database name (for postgres)')
65+ database.aptfile_cachedir = StringOption(
66+ default="~/.cache/pkgme-devportal",
67+ help='The cache directory for the aptfile backend')
68+ database.base_url = StringOption(
69+ default='https://libdep-service.ubuntu.com/',
70+ help='The base URL for libdep-service')
71+
72+ scan_mode = StringOption(
73+ help='Deprecated option, only binary is supported..',
74+ default='binary')
75+
76+ # overrides
77+ libraries = Section()
78+ default_lib_overrides = {
79+ 'libasound.so.2': 'libasound2',
80+ 'libGL.so.1': 'libgl1-mesa-glx',
81+ }
82+ libraries.overrides = DictOption(
83+ default=default_lib_overrides,
84+ help='mapping of library name to pkgname to force picking selected '
85+ 'dependencies')
86+
87+ # The architectures that we fetch binary packages for, add symbols
88+ # to the database and support creating debian packages for.
89+ architectures = Section()
90+ architectures.supported = TupleOption(
91+ # XXX: mvo: it seems we don't need "all" here as we are interessted
92+ # in binary symbols only?
93+ default=("i386", "amd64"),
94+ help='The architectures that we look at for adding libraries to '
95+ 'our database and that we can build packages for')
96+
97+
98+def get_config_file_path():
99+ """Return the path to the configuration file."""
100+ from_env = os.environ.get(CONF_FILE_ENV_VAR, None)
101+ if from_env:
102+ return from_env
103+ return os.path.expanduser(_DEFAULT_CONF_FILE)
104+
105+
106+class Options(object):
107+
108+ def set(self, name, value):
109+ setattr(self, name, value)
110+
111+
112+def load_configuration():
113+ config_location = get_config_file_path()
114+ config_files = []
115+ if os.path.exists(config_location):
116+ config_files.append(config_location)
117+ schema_parser = parser.SchemaConfigParser(DevportalSchema())
118+ # tell the SchemaConfigParser that we need our data case-sensitive
119+ schema_parser.optionxform = str
120+ schema_parser.read(config_files)
121+ result = Options()
122+ for section, data in schema_parser.values().items():
123+ for option, value in data.items():
124+ result.set('{}_{}'.format(section, option), value)
125+ return result
126
127=== added file 'djlibdep/database.py'
128--- djlibdep/database.py 1970-01-01 00:00:00 +0000
129+++ djlibdep/database.py 2012-11-12 17:37:20 +0000
130@@ -0,0 +1,168 @@
131+
132+
133+from storm.expr import And, Column, Select, Table
134+from storm.locals import create_database, Store
135+from storm.uri import URI as StormURI
136+
137+from .configuration import (
138+ CONF_FILE_ENV_VAR,
139+ get_config_file_path,
140+ load_configuration,
141+)
142+
143+
144+class URI(StormURI):
145+ """A stand-in for Storm's URI class.
146+
147+ This class implements the same interface as `storm.uri.URI`, except
148+ that the constructor has a different signature. Storm's version takes
149+ a string and parses it, this version can be used when you already
150+ have a parsed version and just need to create the object.
151+ """
152+
153+ def __init__(self, scheme=None, host=None, port=None, username=None,
154+ password=None, database=None, options=None):
155+ self.scheme = scheme
156+ self.host = host
157+ self.port = port
158+ self.username = username
159+ self.password = password
160+ self.database = database
161+ self.options = options
162+ if self.options is None:
163+ self.options = dict()
164+
165+
166+class PackageDatabase(object):
167+
168+ SQLITE = 'sqlite'
169+ POSTGRES = 'postgres'
170+
171+ def __init__(self, store):
172+ self._store = store
173+
174+ @classmethod
175+ def _get_storm_sqlite_connection_uri(cls, opts):
176+ raise ValueError(
177+ "SQLite is no longer supported, you must migrate to postgresql.")
178+
179+ @classmethod
180+ def _get_storm_postgres_connection_uri(cls, opts):
181+ if not getattr(opts, 'database_db_name', None):
182+ raise ValueError(
183+ "Can't create database, no connection info available. "
184+ "You must specify %s. Looked in %s. "
185+ "Perhaps %s is set incorrectly?" % (
186+ 'db_name', get_config_file_path(), CONF_FILE_ENV_VAR))
187+ return URI(scheme=opts.database_db_type,
188+ username=opts.database_username,
189+ password=opts.database_password,
190+ host=opts.database_host,
191+ port=opts.database_port,
192+ database=opts.database_db_name)
193+
194+ @classmethod
195+ def _get_storm_connection_uri(cls, opts):
196+ if opts.database_db_type == cls.POSTGRES:
197+ return cls._get_storm_postgres_connection_uri(opts)
198+ elif opts.database_db_type == cls.SQLITE:
199+ return cls._get_storm_sqlite_connection_uri(opts)
200+ else:
201+ raise AssertionError(
202+ "Unsupported database: %s" % opts.database_db_type)
203+
204+ @classmethod
205+ def get_db_info_from_config(cls, opts):
206+ return cls._get_storm_connection_uri(opts)
207+
208+ @classmethod
209+ def get_store_from_config(cls, opts):
210+ """Create a storm store based on a config file.
211+
212+ This method will create a storm store based
213+ on the information in ``~/.config/pkgme-binary/conf``
214+
215+ :return: a tuple of (store, store_type), where store_type
216+ is one of cls.SQLITE or cls.POSTGRES, indicating what
217+ is at the other end of the store.
218+ """
219+ connection_info = cls.get_db_info_from_config(opts)
220+ database = create_database(connection_info)
221+ return Store(database)
222+
223+ @classmethod
224+ def from_options(cls, options):
225+ return cls(cls.get_store_from_config(options))
226+
227+ def _get_query(self, library_names, arch):
228+ return Select(
229+ [Column('library'), Column('dependency')],
230+ And(Column('architecture') == arch,
231+ Column('library').is_in(map(unicode, library_names))),
232+ Table('libdep'))
233+
234+ def get_multiple_dependencies(self, library_names, arch):
235+ """Get the binary packages that provide libraries.
236+
237+ :return: (deps, missing), where ``deps`` is a dict mapping library
238+ names to sets of packages that provide them, and ``missing`` is a
239+ set of library names for which no dependencies could be found.
240+ """
241+ arch = unicode(arch)
242+ result = self._store.execute(self._get_query(library_names, arch))
243+ found = {}
244+ for row in result:
245+ [lib, dependency] = row
246+ if lib in found:
247+ found[lib].add(dependency)
248+ else:
249+ found[lib] = set([dependency])
250+ return found
251+
252+ def insert_new_library(self, package_name, library_name,
253+ dependency, arch):
254+ """Insert a library and its needed dependency into the database.
255+
256+ :param library_name: A full soname, e.g. libfoo.so.1.
257+ :param dependency: A binary package dependency, possibly including
258+ version.
259+ """
260+ self._store.execute(
261+ "INSERT INTO libdep VALUES (?, ?, ?, ?)",
262+ (unicode(package_name),
263+ unicode(library_name),
264+ unicode(dependency),
265+ unicode(arch)))
266+
267+ def update_package(self, package_name, arch_libdep_mapping):
268+ """Update the database with the libdep info from 'package_name'.
269+
270+ :param package_name: The name of the package where the
271+ symbols came from.
272+ :param arch_libdep_mapping: a dict mapping architecture tags to dicts
273+ mapping library names to dependencies, e.g.
274+ {'amd64': {'libfoo.so.1': 'libfoo1', ...}, ...}
275+ """
276+ for arch, libdep_mapping in arch_libdep_mapping.items():
277+ self._store.execute(
278+ "DELETE FROM libdep WHERE source_package_name = ? "
279+ "AND architecture = ?",
280+ (unicode(package_name), unicode(arch)))
281+ for library, dependency in libdep_mapping.items():
282+ self.insert_new_library(
283+ package_name, library, dependency, arch)
284+ self._store.commit()
285+
286+ def close(self):
287+ self._store.close()
288+
289+
290+def get_dependency_database():
291+ """Return an object that can get dependencies."""
292+ # TODO: Change this to return LibdepServiceClient sometimes
293+ databases = {
294+ PackageDatabase.POSTGRES: PackageDatabase.from_options,
295+ PackageDatabase.SQLITE: PackageDatabase.from_options,
296+ }
297+ options = load_configuration()
298+ return databases[options.database_db_type](options)
299
300=== added directory 'djlibdep/db'
301=== added file 'djlibdep/db/patch-00001.sql'
302--- djlibdep/db/patch-00001.sql 1970-01-01 00:00:00 +0000
303+++ djlibdep/db/patch-00001.sql 2012-11-12 17:37:20 +0000
304@@ -0,0 +1,3 @@
305+
306+ALTER TABLE libdep
307+ ADD COLUMN architecture TEXT;
308
309=== added file 'djlibdep/db/patch-00002.sql'
310--- djlibdep/db/patch-00002.sql 1970-01-01 00:00:00 +0000
311+++ djlibdep/db/patch-00002.sql 2012-11-12 17:37:20 +0000
312@@ -0,0 +1,11 @@
313+ALTER TABLE libdep
314+ ALTER COLUMN architecture SET NOT NULL;
315+
316+-- also update the unique constraint
317+
318+ALTER TABLE libdep
319+ DROP CONSTRAINT libdep_uniq;
320+
321+ALTER TABLE libdep
322+ ADD CONSTRAINT libdep_uniq
323+ UNIQUE (source_package_name, library, dependency, architecture);
324
325=== added file 'djlibdep/db/postgres_schema.sql'
326--- djlibdep/db/postgres_schema.sql 1970-01-01 00:00:00 +0000
327+++ djlibdep/db/postgres_schema.sql 2012-11-12 17:37:20 +0000
328@@ -0,0 +1,7 @@
329+CREATE TABLE IF NOT EXISTS libdep (
330+ source_package_name TEXT,
331+ library TEXT,
332+ dependency TEXT,
333+ CONSTRAINT libdep_uniq UNIQUE (
334+ source_package_name, library, dependency)
335+);
336
337=== modified file 'djlibdep/tasks.py'
338--- djlibdep/tasks.py 2012-10-30 10:27:00 +0000
339+++ djlibdep/tasks.py 2012-11-12 17:37:20 +0000
340@@ -18,7 +18,7 @@
341 task,
342 )
343
344-from devportalbinary.database import get_dependency_database
345+from .database import get_dependency_database
346
347 from .api import update_packages
348 from .aptfile import iter_library_packages
349
350=== added file 'djlibdep/testing.py'
351--- djlibdep/testing.py 1970-01-01 00:00:00 +0000
352+++ djlibdep/testing.py 2012-11-12 17:37:20 +0000
353@@ -0,0 +1,192 @@
354+from contextlib import closing
355+import os
356+
357+from fixtures import (
358+ EnvironmentVariableFixture,
359+ Fixture,
360+ TempDir,
361+ )
362+from postgresfixture import ClusterFixture
363+from storm.locals import create_database, Store
364+from testresources import (
365+ FixtureResource as _FixtureResource,
366+ )
367+
368+from .configuration import CONF_FILE_ENV_VAR
369+from .database import PackageDatabase, URI
370+
371+
372+def get_db_schema_file_path(name):
373+ return os.path.join(os.path.dirname(
374+ os.path.abspath(__file__)), 'db', name)
375+
376+
377+def get_db_schema_queries(filenames):
378+ for filename in filenames:
379+ path = get_db_schema_file_path(filename)
380+ with open(path) as f:
381+ yield f.read()
382+
383+
384+class PostgresDatabaseFixture(Fixture):
385+
386+ def __init__(self):
387+ super(PostgresDatabaseFixture, self).__init__()
388+ self.db_name = "libdep"
389+
390+ def drop_db(self):
391+ # stub suggests that dropping all tables would be quicker than
392+ # dropping the db when the number of tables is small.
393+ # select quote_ident(table_schema) || '.' ||
394+ # quote_ident(table_name) from information_schema.tables
395+ # WHERE table_schema = 'public';
396+ self.cluster.dropdb(self.db_name)
397+
398+ def create_db(self):
399+ self.cluster.createdb(self.db_name)
400+ queries = [
401+ 'postgres_schema.sql',
402+ 'patch-00001.sql',
403+ 'patch-00002.sql',
404+ ]
405+ for patch in get_db_schema_queries(queries):
406+ self._execute(patch)
407+
408+ def _execute(self, query):
409+ with closing(self.cluster.connect(self.db_name)) as conn:
410+ cur = conn.cursor()
411+ cur.execute(query)
412+ conn.commit()
413+
414+ def close_connection(self):
415+ self.conn.close()
416+
417+ def open_connection(self):
418+ db = create_database(
419+ URI(scheme='postgres', host=self.cluster.datadir,
420+ database=self.db_name))
421+ self.conn = Store(db)
422+ self.addCleanup(self.close_connection)
423+
424+ def reset(self):
425+ self.close_connection()
426+ self.drop_db()
427+ self.create_db()
428+ self.open_connection()
429+
430+ def setUp(self):
431+ super(PostgresDatabaseFixture, self).setUp()
432+ self.tempdir = self.useFixture(TempDir())
433+ self.cluster = self.useFixture(ClusterFixture(self.tempdir.path))
434+ self.create_db()
435+ self.open_connection()
436+
437+
438+class FixtureResource(_FixtureResource):
439+ """The built in FixtureResource doesn't get properly dirtied."""
440+ # XXX: workaround for bug 1023423
441+
442+ def _get_dirty(self):
443+ return True
444+
445+ def _set_dirty(self, new_val):
446+ pass
447+
448+ _dirty = property(_get_dirty, _set_dirty)
449+
450+
451+class PostgresDatabaseResource(FixtureResource):
452+
453+ def __init__(self):
454+ fixture = PostgresDatabaseFixture()
455+ super(PostgresDatabaseResource, self).__init__(fixture)
456+
457+ def reset(self, resource, result=None):
458+ resource.reset()
459+ return resource
460+
461+
462+postgres_db_resource = PostgresDatabaseResource()
463+
464+
465+class DatabaseConfig(Fixture):
466+
467+ def __init__(self, db_fixture):
468+ super(DatabaseConfig, self).__init__()
469+ self.db_fixture = db_fixture
470+
471+ def setUp(self):
472+ super(DatabaseConfig, self).setUp()
473+ self.useFixture(
474+ ConfigSettings(
475+ ('database', {'db_type': 'postgres',
476+ 'host': self.db_fixture.cluster.datadir,
477+ 'db_name': self.db_fixture.db_name,
478+ })))
479+
480+
481+class DatabaseFixture(Fixture):
482+ """Create a temporary database and make it the default.
483+
484+ Don't use this twice within a test, otherwise you'll get confused.
485+ """
486+
487+ def setUp(self):
488+ super(DatabaseFixture, self).setUp()
489+ pg_db = self.useFixture(PostgresDatabaseFixture())
490+ self.useFixture(DatabaseConfig(pg_db))
491+ self.db = PackageDatabase(pg_db.conn)
492+ self.addCleanup(self.db.close)
493+
494+
495+def ConfigFileFixture(location):
496+ """Use a different configuration file."""
497+ return EnvironmentVariableFixture(CONF_FILE_ENV_VAR, location)
498+
499+
500+class ConfigSettings(Fixture):
501+ """Use a configuration file with different settings."""
502+
503+ def __init__(self, *settings):
504+ """Construct a `ConfigSettings` fixture.
505+
506+ :param *settings: A list of tuples ``(section, values)`` where
507+ ``section`` is the name of the configuration section and
508+ ``values`` is a dict mapping individual settings to their
509+ values.
510+ """
511+ super(ConfigSettings, self).__init__()
512+ self._settings = settings
513+
514+ def setUp(self):
515+ super(ConfigSettings, self).setUp()
516+ # Set a temporary homedir so that any config on the user's
517+ # machine isn't picked up and the environment variable is used
518+ # instead.
519+ tempdir = self.useFixture(TempDir())
520+ config_file_path = os.path.join(tempdir.path, 'test.cfg')
521+ write_config_file(config_file_path, self._settings)
522+ self.useFixture(ConfigFileFixture(config_file_path))
523+
524+
525+def make_config_section(key, values):
526+ lines = ['[%s]' % (key,)]
527+ for key, value in values.items():
528+ lines.append('%s=%s' % (key, value))
529+ lines.append('')
530+ return '\n'.join(lines)
531+
532+
533+def write_config_file(config_file_path, settings):
534+ """Write a config file to ``config_file_path``.
535+
536+ :param config_file_path: The path to write a new config file to.
537+ :param settings: A list of tuples ``(section, values)`` where
538+ ``section`` is the name of the configuration section and
539+ ``values`` is a dict mapping individual settings to their
540+ values.
541+ """
542+ with open(config_file_path, 'w') as f:
543+ for section, values in settings:
544+ f.write(make_config_section(section, values))
545+ f.write('\n')
546
547=== modified file 'djlibdep/tests/helpers.py'
548--- djlibdep/tests/helpers.py 2012-09-13 22:20:17 +0000
549+++ djlibdep/tests/helpers.py 2012-11-12 17:37:20 +0000
550@@ -12,8 +12,8 @@
551 # You should have received a copy of the GNU Affero General Public License
552 # along with this program. If not, see <http://www.gnu.org/licenses/>.
553
554-from devportalbinary.database import PackageDatabase
555-from devportalbinary.testing import (
556+from ..database import PackageDatabase
557+from ..testing import (
558 DatabaseConfig,
559 postgres_db_resource,
560 )
561
562=== modified file 'djlibdep/tests/test_interface.py'
563--- djlibdep/tests/test_interface.py 2012-10-25 14:00:07 +0000
564+++ djlibdep/tests/test_interface.py 2012-11-12 17:37:20 +0000
565@@ -6,11 +6,6 @@
566 urlopen,
567 )
568
569-from devportalbinary.database import PackageDatabase
570-from devportalbinary.testing import (
571- DatabaseConfig,
572- PostgresDatabaseFixture,
573- )
574 from djangofixture import DjangoFixture
575 from fixtures import Fixture
576 from testresources import (
577@@ -20,6 +15,11 @@
578 from testscenarios import generate_scenarios
579 from testtools import TestCase
580
581+from ..database import PackageDatabase
582+from ..testing import (
583+ DatabaseConfig,
584+ PostgresDatabaseFixture,
585+ )
586 from .helpers import populate_sample_data
587 from .test_test_double import (
588 test_double_fixture,
589
590=== modified file 'djlibdep/tests/test_tasks.py'
591--- djlibdep/tests/test_tasks.py 2012-10-30 14:07:32 +0000
592+++ djlibdep/tests/test_tasks.py 2012-11-12 17:37:20 +0000
593@@ -14,7 +14,6 @@
594
595 import os
596
597-from devportalbinary.testing import DatabaseFixture
598 from fixtures import MonkeyPatch
599 from testtools import TestCase
600 from testtools.matchers import GreaterThan
601@@ -28,6 +27,7 @@
602 update_all,
603 update_architecture,
604 )
605+from ..testing import DatabaseFixture
606
607
608 SAMPLE_CONTENTS = os.path.join(os.path.dirname(__file__), 'Contents.gz')
609
610=== modified file 'djlibdep/views.py'
611--- djlibdep/views.py 2012-10-17 14:14:57 +0000
612+++ djlibdep/views.py 2012-11-12 17:37:20 +0000
613@@ -12,7 +12,6 @@
614 # You should have received a copy of the GNU Affero General Public License
615 # along with this program. If not, see <http://www.gnu.org/licenses/>.
616
617-from devportalbinary.database import get_dependency_database
618 from django.http import (
619 HttpResponse,
620 HttpResponseForbidden,
621@@ -21,6 +20,7 @@
622 from django.template import RequestContext
623
624 from . import api
625+from .database import get_dependency_database
626 from .stats import (
627 concurrent_request,
628 timed,

Subscribers

People subscribed via source and target branches