This is a megamerge branch that incorporates bug 309859, bug 309988, bug 310619, and bug 310782, as well as a few additional refinements for the .master section. This is the branch I'm currently using in Mailman 3. === modified file 'src/lazr/config/README.txt' --- src/lazr/config/README.txt 2008-12-15 22:22:12 +0000 +++ src/lazr/config/README.txt 2009-01-04 20:06:34 +0000 @@ -79,6 +79,21 @@ >>> schema.filename '...lazr/config/testdata/base.conf' +If you provide an optional file-like object as a second argument to the +constructor, that is used instead of opening the named file implicitly. + + >>> file_object = open(base_conf) + >>> other_schema = ConfigSchema('/does/not/exist.conf', file_object) + >>> verifyObject(IConfigSchema, other_schema) + True + + >>> print other_schema.name + exist.conf + >>> print other_schema.filename + /does/not/exist.conf + + >>> file_object.close() + A schema is made up of multiple SchemaSections. They can be iterated over in a loop as needed. @@ -92,6 +107,15 @@ section_3.app_b section_33 + >>> for section_schema in sorted(other_schema, key=attrgetter('name')): + ... print section_schema.name + section-2.app-b + section-5 + section_1 + section_3.app_a + section_3.app_b + section_33 + You can check if the schema contains a section name, and that can be used to access the SchemaSection as a subscript. @@ -139,6 +163,13 @@ ... NoCategoryError: ... +You can pass a default argument to getByCategory() to avoid the exception. + + >>> missing = object() + >>> schema.getByCategory('non-section', missing) is missing + True + + ============= SchemaSection ============= @@ -368,6 +399,27 @@ # Accept the default values for the optional section-5. [section-5] +The .master section allows admins to define configurations for an arbitrary +number of processes. If the schema defines .master sections, then the conf +file can contain sections that extend the .master section. These are like +categories with templates except that the section names extending .master need +not be named in the schema file. + + >>> master_schema_conf = path.join(testfiles_dir, 'master.conf') + >>> master_local_conf = path.join(testfiles_dir, 'master-local.conf') + >>> master_schema = ConfigSchema(master_schema_conf) + >>> sections = master_schema.getByCategory('thing') + >>> sorted(section.name for section in sections) + ['thing.master'] + >>> master_conf = master_schema.load(master_local_conf) + >>> sections = master_conf.getByCategory('thing') + >>> sorted(section.name for section in sections) + ['thing.one', 'thing.two'] + >>> sorted(section.foo for section in sections) + ['1', '2'] + >>> print master_conf.thing.one.name + thing.one + The shared.conf file derives the keys and default values from the schema. This config was loaded before local.conf because its sections and values are required to be in place before local.conf applies its @@ -481,6 +533,13 @@ ... NoCategoryError: ... +As with schemas, you can pass a default argument to getByCategory() to avoid +the exception. + + >>> missing = object() + >>> config.getByCategory('non-section', missing) is missing + True + ======= Section ======= @@ -774,6 +833,66 @@ key4 : Fc;k yeah! key5 : +push() can also be used to extend master sections. + + >>> sections = sorted(master_conf.getByCategory('bar'), + ... key=attrgetter('name')) + >>> for section in sections: + ... print section.name, section.baz + bar.master badger + bar.soup cougar + + >>> master_conf.push('override', """ + ... [bar.two] + ... baz: dolphin + ... """) + >>> sections = sorted(master_conf.getByCategory('bar'), + ... key=attrgetter('name')) + >>> for section in sections: + ... print section.name, section.baz + bar.soup cougar + bar.two dolphin + + >>> master_conf.push('overlord', """ + ... [bar.three] + ... baz: emu + ... """) + >>> sections = sorted(master_conf.getByCategory('bar'), + ... key=attrgetter('name')) + >>> for section in sections: + ... print section.name, section.baz + bar.soup cougar + bar.three emu + bar.two dolphin + +push() works with master sections too. + + >>> schema_file = StringIO.StringIO("""\ + ... [thing.master] + ... foo: 0 + ... bar: 0 + ... """) + >>> push_schema = ConfigSchema('schema.cfg', schema_file) + + >>> config_file = StringIO.StringIO("""\ + ... [thing.one] + ... foo: 1 + ... """) + >>> push_config = push_schema.loadFile(config_file, 'config.cfg') + >>> print push_config.thing.one.foo + 1 + >>> print push_config.thing.one.bar + 0 + + >>> push_config.push('test.cfg', """\ + ... [thing.one] + ... bar: 2 + ... """) + >>> print push_config.thing.one.foo + 1 + >>> print push_config.thing.one.bar + 2 + pop() ===== @@ -1048,6 +1167,60 @@ functions have to be imported and called explicitly on the configuration variable values. +Booleans +======== + +There is a helper for turning various strings into the boolean values True and +False. + + >>> from lazr.config import as_boolean + +True values include (case-insensitively): true, yes, 1, on, enabled, and +enable. + + >>> for value in ('true', 'yes', 'on', 'enable', 'enabled', '1'): + ... print value, '->', as_boolean(value) + ... print value.upper(), '->', as_boolean(value.upper()) + true -> True + TRUE -> True + yes -> True + YES -> True + on -> True + ON -> True + enable -> True + ENABLE -> True + enabled -> True + ENABLED -> True + 1 -> True + 1 -> True + +False values include (case-insensitively): false, no, 0, off, disabled, and +disable. + + >>> for value in ('false', 'no', 'off', 'disable', 'disabled', '0'): + ... print value, '->', as_boolean(value) + ... print value.upper(), '->', as_boolean(value.upper()) + false -> False + FALSE -> False + no -> False + NO -> False + off -> False + OFF -> False + disable -> False + DISABLE -> False + disabled -> False + DISABLED -> False + 0 -> False + 0 -> False + +Anything else is a error. + + >>> as_boolean('cheese') + Traceback (most recent call last): + ... + ValueError: Invalid boolean value: cheese + + Host and port ============= @@ -1096,7 +1269,7 @@ >>> as_host_port(':foo') Traceback (most recent call last): ... - ValueError: invalid literal for int(): foo + ValueError: invalid literal for int...foo... User and group ============== @@ -1209,3 +1382,37 @@ Traceback (most recent call last): ... ValueError + +Log levels +========== + +It's convenient to be able to use symbolic log level names when using +lazr.config to configure the Python logger. + + >>> from lazr.config import as_log_level + +Any symbolic log level value is valid to use, case insensitively. + + >>> for value in ('critical', 'error', 'warning', 'info', + ... 'debug', 'notset'): + ... print value, '->', as_log_level(value) + ... print value.upper(), '->', as_log_level(value.upper()) + critical -> 50 + CRITICAL -> 50 + error -> 40 + ERROR -> 40 + warning -> 30 + WARNING -> 30 + info -> 20 + INFO -> 20 + debug -> 10 + DEBUG -> 10 + notset -> 0 + NOTSET -> 0 + +Non-log levels cannot be used here. + + >>> as_log_level('cheese') + Traceback (most recent call last): + ... + AttributeError: 'module' object has no attribute 'CHEESE' === modified file 'src/lazr/config/__init__.py' --- src/lazr/config/__init__.py 2008-12-15 22:22:12 +0000 +++ src/lazr/config/__init__.py 2009-01-04 20:06:34 +0000 @@ -12,7 +12,9 @@ 'ImplicitTypeSection', 'Section', 'SectionSchema', + 'as_boolean', 'as_host_port', + 'as_log_level', 'as_timedelta', 'as_username_groupname', ] @@ -21,6 +23,7 @@ import StringIO import datetime import grp +import logging import os import pwd import re @@ -38,6 +41,8 @@ UnknownSectionError) from lazr.delegates import delegates +_missing = object() + def read_content(filename): """Return the content of a file at filename as a string.""" @@ -53,7 +58,7 @@ """See `ISectionSchema`.""" implements(ISectionSchema) - def __init__(self, name, options, is_optional=False): + def __init__(self, name, options, is_optional=False, is_master=False): """Create an `ISectionSchema` from the name and options. :param name: A string. The name of the ISectionSchema. @@ -66,6 +71,7 @@ self.name = name self._options = options self.optional = is_optional + self.master = is_master def __iter__(self): """See `ISectionSchema`""" @@ -87,9 +93,15 @@ else: return (None, self.name) + def clone(self): + """Return a copy of this section schema.""" + return self.__class__(self.name, self._options.copy(), + self.optional, self.master) + class Section: """See `ISection`.""" + implements(ISection) delegates(ISectionSchema, context='schema') @@ -101,7 +113,7 @@ # Use __dict__ because __getattr__ limits access to self.options. self.__dict__['schema'] = schema if _options is None: - _options = dict([(key, schema[key]) for key in schema]) + _options = dict((key, schema[key]) for key in schema) self.__dict__['_options'] = _options def __getitem__(self, key): @@ -146,14 +158,7 @@ The extension mechanism requires a copy of a section to prevent mutation. """ - new_section = self.__class__(self.schema, self._options.copy()) - # XXX 2008-06-10 jamesh bug=237827: - # Evil legacy code sometimes assigns directly to the config - # section objects. Copy those attributes over. - new_section.__dict__.update( - dict((key, value) for (key, value) in self.__dict__.iteritems() - if key not in ['schema', '_options'])) - return new_section + return self.__class__(self.schema, self._options.copy()) class ImplicitTypeSection(Section): @@ -210,9 +215,15 @@ _section_factory = Section - def __init__(self, filename): + def __init__(self, filename, file_object=None): """Load a configuration schema from the provided filename. + :param filename: The name of the file to load from, or if + `file_object` is given, to pretend to load from. + :type filename: string + :param file_object: If given, optional file-like object to read from + instead of actually opening the named file. + :type file_object: An object with a readline() method. :raise `UnicodeDecodeError`: if the string contains non-ascii characters. :raise `RedefinedSectionError`: if a SectionSchema name is redefined. @@ -226,12 +237,14 @@ self.name = basename(filename) self._section_schemas = {} self._category_names = [] - raw_schema = self._getRawSchema(filename) + if file_object is None: + raw_schema = self._getRawSchema(filename) + else: + raw_schema = file_object parser = RawConfigParser() parser.readfp(raw_schema, filename) self._setSectionSchemasAndCategoryNames(parser) - def _getRawSchema(self, filename): """Return the contents of the schema at filename as a StringIO. @@ -254,22 +267,24 @@ """Set the SectionSchemas and category_names from the config.""" category_names = set() templates = {} - # Retrieve all the templates first because section() does not - # follow the order of the conf file. + # Retrieve all the templates first because section() does not follow + # the order of the conf file. for name in parser.sections(): (section_name, category_name, - is_template, is_optional) = self._parseSectionName(name) - if is_template: + is_template, is_optional, + is_master) = self._parseSectionName(name) + if is_template or is_master: templates[category_name] = dict(parser.items(name)) for name in parser.sections(): (section_name, category_name, - is_template, is_optional) = self._parseSectionName(name) + is_template, is_optional, + is_master) = self._parseSectionName(name) if is_template: continue options = dict(templates.get(category_name, {})) options.update(parser.items(name)) self._section_schemas[section_name] = SectionSchema( - section_name, options, is_optional) + section_name, options, is_optional, is_master) if category_name is not None: category_names.add(category_name) self._category_names = list(category_names) @@ -277,7 +292,7 @@ _section_name_pattern = re.compile(r'\w[\w.-]+\w') def _parseSectionName(self, name): - """Return a 4-tuple of names and kinds embedded in the name. + """Return a tuple of names and kinds embedded in the name. :return: (section_name, category_name, is_template, is_optional). section_name is always a string. category_name is a string or @@ -288,6 +303,7 @@ name_parts = name.split('.') is_template = name_parts[-1] == 'template' is_optional = name_parts[-1] == 'optional' + is_master = name_parts[-1] == 'master' if is_template or is_optional: # The suffix is not a part of the section name. # Example: [name.optional] or [category.template] @@ -310,7 +326,8 @@ if self._section_name_pattern.match(section_name) is None: raise InvalidSectionNameError( '[%s] name does not match [\w.-]+.' % name) - return (section_name, category_name, is_template, is_optional) + return (section_name, category_name, + is_template, is_optional, is_master) @property def section_factory(self): @@ -337,10 +354,12 @@ except KeyError: raise NoSectionError(name) - def getByCategory(self, name): + def getByCategory(self, name, default=_missing): """See `IConfigSchema`.""" if name not in self.category_names: - raise NoCategoryError(name) + if default is _missing: + raise NoCategoryError(name) + return default section_schemas = [] for key in self._section_schemas: section = self._section_schemas[key] @@ -436,10 +455,12 @@ except KeyError: raise NoSectionError(name) - def getByCategory(self, name): + def getByCategory(self, name, default=_missing): """See `IConfigData`.""" if name not in self.category_names: - raise NoCategoryError(name) + if default is _missing: + raise NoCategoryError(name) + return default sections = [] for key in self._sections: section = self._sections[key] @@ -515,9 +536,9 @@ self._overlays = (config_data, ) + self._overlays def _getExtendedConfs(self, conf_filename, conf_data, confs=None): - """Return a list of 3-tuple(conf_name, parser, encoding_errors). + """Return a list of tuple (conf_name, parser, encoding_errors). - :param conf_filename: The path and name the conf file. + :param conf_filename: The path and name of the conf file. :param conf_data: Unparsed config data. :param confs: A list of confs that extend filename. :return: A list of confs ordered from extender to extendee. @@ -551,7 +572,7 @@ :return: a new ConfigData object. This method extracts the sections, keys, and values from the parser - to construct a new ConfigData object The list of encoding errors are + to construct a new ConfigData object. The list of encoding errors are incorporated into the the list of data-related errors for the ConfigData. """ @@ -561,22 +582,48 @@ errors = list(self.data._errors) errors.extend(encoding_errors) extends = None + masters = set() for section_name in parser.sections(): if section_name == 'meta': extends, meta_errors = self._loadMetaData(parser) errors.extend(meta_errors) continue - if (section_name.endswith('.template') - or section_name.endswith('.optional')): + if (section_name.endswith('.template') or + section_name.endswith('.optional') or + section_name.endswith('.master')): # This section is a schema directive. continue - if section_name not in self.schema: + # Calculate the section master name. + # Check for sections which extend .masters. + if '.' in section_name: + category, section = section_name.split('.') + master_name = category + '.master' + else: + master_name = None + if (section_name not in self.schema and + master_name not in self.schema): # Any section not in the the schema is an error. msg = "%s does not have a %s section." % ( self.schema.name, section_name) errors.append(UnknownSectionError(msg)) continue if section_name not in self.data: + # Is there a master section? + try: + section_schema = self.schema[master_name] + except NoSectionError: + # There's no master for this section, so just treat it + # like a regular category. + pass + else: + assert section_schema.master, '.master is not a master?' + schema = section_schema.clone() + schema.name = section_name + section = self.schema.section_factory(schema) + section.update(parser.items(section_name)) + sections[section_name] = section + masters.add(master_name) + continue # Create the optional section from the schema. section_schema = self.schema[section_name] sections[section_name] = self.schema.section_factory( @@ -585,6 +632,10 @@ items = parser.items(section_name) section_errors = sections[section_name].update(items) errors.extend(section_errors) + # master sections are like templates. They show up in the schema but + # not in the config. + for master in masters: + sections.pop(master, None) return ConfigData(conf_name, sections, extends, errors) def _verifyEncoding(self, config_data): @@ -660,6 +711,25 @@ raise AttributeError("No section named %s." % name) +def as_boolean(value): + """Turn a string into a boolean. + + :param value: A string with one of the following values + (case-insensitive): true, yes, 1, on, enable, enabled (for True), or + false, no, 0, off, disable, disabled (for False). Everything else is + an error. + :type value: string + :return: True or False. + :rtype: boolean + """ + value = value.lower() + if value in ('true', 'yes', '1', 'on', 'enabled', 'enable'): + return True + if value in ('false', 'no', '0', 'off', 'disabled', 'disable'): + return False + raise ValueError('Invalid boolean value: %s' % value) + + def as_host_port(value, default_host='localhost', default_port=25): """Return a 2-tuple of (host, port) from a value like 'host:port'. @@ -752,3 +822,15 @@ if len(keyword_arguments) == 0: raise ValueError return datetime.timedelta(**keyword_arguments) + +def as_log_level(value): + """Turn a string into a log level. + + :param value: A string with a value (case-insensitive) equal to one of the + symbolic logging levels. + :type value: string + :return: A logging level constant. + :rtype: int + """ + value = value.upper() + return getattr(logging, value) === modified file 'src/lazr/config/interfaces.py' --- src/lazr/config/interfaces.py 2008-12-15 22:22:12 +0000 +++ src/lazr/config/interfaces.py 2009-01-04 20:06:34 +0000 @@ -22,8 +22,12 @@ 'UnknownKeyError', 'UnknownSectionError'] +from warnings import filterwarnings from zope.interface import Interface, Attribute +# Ignore Python 2.6 deprecation warnings. +filterwarnings('ignore', category=DeprecationWarning, module=r'lazr\.config') + class ConfigSchemaError(Exception): """A base class of all `IConfigSchema` errors.""" @@ -67,6 +71,7 @@ :param message: a message string :param errors: a list of errors in the config, or None """ + # Without the suppression above, this produces a warning in Python 2.6. self.message = message self.errors = errors @@ -128,9 +133,11 @@ The config file contains sections enclosed in square brackets ([]). The section name may be divided into major and minor categories using a dot (.). Beneath each section is a list of key-value pairs, separated - by a colon (:). Multiple sections with the same major category may have - their keys defined in another section that appends the '.template' - suffix to the category name. A section with '.optional' suffix is not + by a colon (:). + + Multiple sections with the same major category may have their keys defined + in another section that appends the '.template' or '.master' suffixes to + the category name. A section with '.optional' suffix is not required. Lines that start with a hash (#) are comments. """ name = Attribute('The basename of the config filename.') === added file 'src/lazr/config/testdata/master-local.conf' --- src/lazr/config/testdata/master-local.conf 1970-01-01 00:00:00 +0000 +++ src/lazr/config/testdata/master-local.conf 2009-01-04 20:06:34 +0000 @@ -0,0 +1,6 @@ +# Define a few categories based on the master. +[thing.one] +foo: 1 + +[thing.two] +foo: 2 === added file 'src/lazr/config/testdata/master.conf' --- src/lazr/config/testdata/master.conf 1970-01-01 00:00:00 +0000 +++ src/lazr/config/testdata/master.conf 2009-01-04 20:06:34 +0000 @@ -0,0 +1,9 @@ +# This section defines a category master. +[thing.master] +foo: aardvark + +[bar.master] +baz: badger + +[bar.soup] +baz: cougar