Issue 10499: Modular interpolation in configparser
diff --git a/Lib/configparser.py b/Lib/configparser.py
index f9bb32c..56c02ba 100644
--- a/Lib/configparser.py
+++ b/Lib/configparser.py
@@ -4,23 +4,13 @@
and followed by "name: value" entries, with continuations and such in
the style of RFC 822.
-The option values can contain format strings which refer to other values in
-the same section, or values in a special [DEFAULT] section.
-
-For example:
-
- something: %(dir)s/whatever
-
-would resolve the "%(dir)s" to the value of dir. All reference
-expansions are done late, on demand.
-
Intrinsic defaults can be specified by passing them into the
-ConfigParser constructor as a dictionary.
+SafeConfigParser constructor as a dictionary.
class:
-ConfigParser -- responsible for parsing a list of
- configuration files, and managing the parsed database.
+SafeConfigParser -- responsible for parsing a list of
+ configuration files, and managing the parsed database.
methods:
@@ -316,7 +306,7 @@
def filename(self):
"""Deprecated, use `source'."""
warnings.warn(
- "This 'filename' attribute will be removed in future versions. "
+ "The 'filename' attribute will be removed in future versions. "
"Use 'source' instead.",
DeprecationWarning, stacklevel=2
)
@@ -362,6 +352,204 @@
_UNSET = object()
+class Interpolation:
+ """Dummy interpolation that passes the value through with no changes."""
+
+ def before_get(self, parser, section, option, value, defaults):
+ return value
+
+ def before_set(self, parser, section, option, value):
+ return value
+
+ def before_read(self, parser, section, option, value):
+ return value
+
+ def before_write(self, parser, section, option, value):
+ return value
+
+
+class BasicInterpolation(Interpolation):
+ """Interpolation as implemented in the classic SafeConfigParser.
+
+ The option values can contain format strings which refer to other values in
+ the same section, or values in the special default section.
+
+ For example:
+
+ something: %(dir)s/whatever
+
+ would resolve the "%(dir)s" to the value of dir. All reference
+ expansions are done late, on demand. If a user needs to use a bare % in
+ a configuration file, she can escape it by writing %%. Other other % usage
+ is considered a user error and raises `InterpolationSyntaxError'."""
+
+ _KEYCRE = re.compile(r"%\(([^)]+)\)s")
+
+ def before_get(self, parser, section, option, value, defaults):
+ L = []
+ self._interpolate_some(parser, option, L, value, section, defaults, 1)
+ return ''.join(L)
+
+ def before_set(self, parser, section, option, value):
+ tmp_value = value.replace('%%', '') # escaped percent signs
+ tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax
+ if '%' in tmp_value:
+ raise ValueError("invalid interpolation syntax in %r at "
+ "position %d" % (value, tmp_value.find('%')))
+ return value
+
+ def _interpolate_some(self, parser, option, accum, rest, section, map,
+ depth):
+ if depth > MAX_INTERPOLATION_DEPTH:
+ raise InterpolationDepthError(option, section, rest)
+ while rest:
+ p = rest.find("%")
+ if p < 0:
+ accum.append(rest)
+ return
+ if p > 0:
+ accum.append(rest[:p])
+ rest = rest[p:]
+ # p is no longer used
+ c = rest[1:2]
+ if c == "%":
+ accum.append("%")
+ rest = rest[2:]
+ elif c == "(":
+ m = self._KEYCRE.match(rest)
+ if m is None:
+ raise InterpolationSyntaxError(option, section,
+ "bad interpolation variable reference %r" % rest)
+ var = parser.optionxform(m.group(1))
+ rest = rest[m.end():]
+ try:
+ v = map[var]
+ except KeyError:
+ raise InterpolationMissingOptionError(
+ option, section, rest, var)
+ if "%" in v:
+ self._interpolate_some(parser, option, accum, v,
+ section, map, depth + 1)
+ else:
+ accum.append(v)
+ else:
+ raise InterpolationSyntaxError(
+ option, section,
+ "'%%' must be followed by '%%' or '(', "
+ "found: %r" % (rest,))
+
+
+class ExtendedInterpolation(Interpolation):
+ """Advanced variant of interpolation, supports the syntax used by
+ `zc.buildout'. Enables interpolation between sections."""
+
+ _KEYCRE = re.compile(r"\$\{([^}]+)\}")
+
+ def before_get(self, parser, section, option, value, defaults):
+ L = []
+ self._interpolate_some(parser, option, L, value, section, defaults, 1)
+ return ''.join(L)
+
+ def before_set(self, parser, section, option, value):
+ tmp_value = value.replace('$$', '') # escaped dollar signs
+ tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax
+ if '$' in tmp_value:
+ raise ValueError("invalid interpolation syntax in %r at "
+ "position %d" % (value, tmp_value.find('%')))
+ return value
+
+ def _interpolate_some(self, parser, option, accum, rest, section, map,
+ depth):
+ if depth > MAX_INTERPOLATION_DEPTH:
+ raise InterpolationDepthError(option, section, rest)
+ while rest:
+ p = rest.find("$")
+ if p < 0:
+ accum.append(rest)
+ return
+ if p > 0:
+ accum.append(rest[:p])
+ rest = rest[p:]
+ # p is no longer used
+ c = rest[1:2]
+ if c == "$":
+ accum.append("$")
+ rest = rest[2:]
+ elif c == "{":
+ m = self._KEYCRE.match(rest)
+ if m is None:
+ raise InterpolationSyntaxError(option, section,
+ "bad interpolation variable reference %r" % rest)
+ path = parser.optionxform(m.group(1)).split(':')
+ rest = rest[m.end():]
+ sect = section
+ opt = option
+ try:
+ if len(path) == 1:
+ opt = path[0]
+ v = map[opt]
+ elif len(path) == 2:
+ sect = path[0]
+ opt = path[1]
+ v = parser.get(sect, opt, raw=True)
+ else:
+ raise InterpolationSyntaxError(
+ option, section,
+ "More than one ':' found: %r" % (rest,))
+ except KeyError:
+ raise InterpolationMissingOptionError(
+ option, section, rest, var)
+ if "$" in v:
+ self._interpolate_some(parser, opt, accum, v, sect,
+ dict(parser.items(sect, raw=True)),
+ depth + 1)
+ else:
+ accum.append(v)
+ else:
+ raise InterpolationSyntaxError(
+ option, section,
+ "'$' must be followed by '$' or '{', "
+ "found: %r" % (rest,))
+
+
+class BrokenInterpolation(Interpolation):
+ """Deprecated interpolation as implemented in the classic ConfigParser.
+ Use BasicInterpolation or ExtendedInterpolation instead."""
+
+ _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+
+ def before_get(self, parser, section, option, value, vars):
+ rawval = value
+ depth = MAX_INTERPOLATION_DEPTH
+ while depth: # Loop through this until it's done
+ depth -= 1
+ if value and "%(" in value:
+ replace = functools.partial(self._interpolation_replace,
+ parser=parser)
+ value = self._KEYCRE.sub(replace, value)
+ try:
+ value = value % vars
+ except KeyError as e:
+ raise InterpolationMissingOptionError(
+ option, section, rawval, e.args[0])
+ else:
+ break
+ if value and "%(" in value:
+ raise InterpolationDepthError(option, section, rawval)
+ return value
+
+ def before_set(self, parser, section, option, value):
+ return value
+
+ @staticmethod
+ def _interpolation_replace(match, parser):
+ s = match.group(1)
+ if s is None:
+ return match.group()
+ else:
+ return "%%(%s)s" % parser.optionxform(s)
+
+
class RawConfigParser(MutableMapping):
"""ConfigParser that does not do interpolation."""
@@ -388,7 +576,8 @@
# space/tab
(?P<value>.*))?$ # everything up to eol
"""
-
+ # Interpolation algorithm to be used if the user does not specify another
+ _DEFAULT_INTERPOLATION = Interpolation()
# Compiled regular expression for matching sections
SECTCRE = re.compile(_SECT_TMPL, re.VERBOSE)
# Compiled regular expression for matching options with typical separators
@@ -406,7 +595,15 @@
allow_no_value=False, *, delimiters=('=', ':'),
comment_prefixes=_COMPATIBLE, strict=False,
empty_lines_in_values=True,
- default_section=DEFAULTSECT):
+ default_section=DEFAULTSECT,
+ interpolation=_UNSET):
+
+ if self.__class__ is RawConfigParser:
+ warnings.warn(
+ "The RawConfigParser class will be removed in future versions."
+ " Use 'SafeConfigParser(interpolation=None)' instead.",
+ DeprecationWarning, stacklevel=2
+ )
self._dict = dict_type
self._sections = self._dict()
self._defaults = self._dict()
@@ -435,7 +632,11 @@
self._strict = strict
self._allow_no_value = allow_no_value
self._empty_lines_in_values = empty_lines_in_values
- self._default_section=default_section
+ if interpolation is _UNSET:
+ self._interpolation = self._DEFAULT_INTERPOLATION
+ else:
+ self._interpolation = interpolation
+ self.default_section=default_section
def defaults(self):
return self._defaults
@@ -451,7 +652,7 @@
Raise DuplicateSectionError if a section by the specified name
already exists. Raise ValueError if name is DEFAULT.
"""
- if section == self._default_section:
+ if section == self.default_section:
raise ValueError('Invalid section name: %s' % section)
if section in self._sections:
@@ -555,7 +756,7 @@
)
self.read_file(fp, source=filename)
- def get(self, section, option, *, vars=None, fallback=_UNSET):
+ def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET):
"""Get an option value for a given section.
If `vars' is provided, it must be a dictionary. The option is looked up
@@ -563,7 +764,12 @@
If the key is not found and `fallback' is provided, it is used as
a fallback value. `None' can be provided as a `fallback' value.
- Arguments `vars' and `fallback' are keyword only.
+ If interpolation is enabled and the optional argument `raw' is False,
+ all interpolations are expanded in the return values.
+
+ Arguments `raw', `vars', and `fallback' are keyword only.
+
+ The section DEFAULT is special.
"""
try:
d = self._unify_values(section, vars)
@@ -574,61 +780,90 @@
return fallback
option = self.optionxform(option)
try:
- return d[option]
+ value = d[option]
except KeyError:
if fallback is _UNSET:
raise NoOptionError(option, section)
else:
return fallback
- def items(self, section):
- try:
- d2 = self._sections[section]
- except KeyError:
- if section != self._default_section:
- raise NoSectionError(section)
- d2 = self._dict()
- d = self._defaults.copy()
- d.update(d2)
- return d.items()
+ if raw or value is None:
+ return value
+ else:
+ return self._interpolation.before_get(self, section, option, value,
+ d)
def _get(self, section, conv, option, **kwargs):
return conv(self.get(section, option, **kwargs))
- def getint(self, section, option, *, vars=None, fallback=_UNSET):
+ def getint(self, section, option, *, raw=False, vars=None,
+ fallback=_UNSET):
try:
- return self._get(section, int, option, vars=vars)
+ return self._get(section, int, option, raw=raw, vars=vars)
except (NoSectionError, NoOptionError):
if fallback is _UNSET:
raise
else:
return fallback
- def getfloat(self, section, option, *, vars=None, fallback=_UNSET):
+ def getfloat(self, section, option, *, raw=False, vars=None,
+ fallback=_UNSET):
try:
- return self._get(section, float, option, vars=vars)
+ return self._get(section, float, option, raw=raw, vars=vars)
except (NoSectionError, NoOptionError):
if fallback is _UNSET:
raise
else:
return fallback
- def getboolean(self, section, option, *, vars=None, fallback=_UNSET):
+ def getboolean(self, section, option, *, raw=False, vars=None,
+ fallback=_UNSET):
try:
return self._get(section, self._convert_to_boolean, option,
- vars=vars)
+ raw=raw, vars=vars)
except (NoSectionError, NoOptionError):
if fallback is _UNSET:
raise
else:
return fallback
+ def items(self, section, raw=False, vars=None):
+ """Return a list of (name, value) tuples for each option in a section.
+
+ All % interpolations are expanded in the return values, based on the
+ defaults passed into the constructor, unless the optional argument
+ `raw' is true. Additional substitutions may be provided using the
+ `vars' argument, which must be a dictionary whose contents overrides
+ any pre-existing defaults.
+
+ The section DEFAULT is special.
+ """
+ d = self._defaults.copy()
+ try:
+ d.update(self._sections[section])
+ except KeyError:
+ if section != self.default_section:
+ raise NoSectionError(section)
+ # Update with the entry specific variables
+ if vars:
+ for key, value in vars.items():
+ d[self.optionxform(key)] = value
+ options = list(d.keys())
+ if raw:
+ return [(option, d[option])
+ for option in options]
+ else:
+ return [(option, self._interpolation.before_get(self, section,
+ option, d[option],
+ d))
+ for option in options]
+
def optionxform(self, optionstr):
return optionstr.lower()
def has_option(self, section, option):
"""Check for the existence of a given option in a given section."""
- if not section or section == self._default_section:
+ if not section or section == self.default_section:
option = self.optionxform(option)
return option in self._defaults
elif section not in self._sections:
@@ -640,7 +875,10 @@
def set(self, section, option, value=None):
"""Set an option."""
- if not section or section == self._default_section:
+ if value:
+ value = self._interpolation.before_set(self, section, option,
+ value)
+ if not section or section == self.default_section:
sectdict = self._defaults
else:
try:
@@ -660,7 +898,7 @@
else:
d = self._delimiters[0]
if self._defaults:
- self._write_section(fp, self._default_section,
+ self._write_section(fp, self.default_section,
self._defaults.items(), d)
for section in self._sections:
self._write_section(fp, section,
@@ -670,6 +908,8 @@
"""Write a single section to the specified `fp'."""
fp.write("[{}]\n".format(section_name))
for key, value in section_items:
+ value = self._interpolation.before_write(self, section_name, key,
+ value)
if value is not None or not self._allow_no_value:
value = delimiter + str(value).replace('\n', '\n\t')
else:
@@ -679,7 +919,7 @@
def remove_option(self, section, option):
"""Remove an option."""
- if not section or section == self._default_section:
+ if not section or section == self.default_section:
sectdict = self._defaults
else:
try:
@@ -701,7 +941,7 @@
return existed
def __getitem__(self, key):
- if key != self._default_section and not self.has_section(key):
+ if key != self.default_section and not self.has_section(key):
raise KeyError(key)
return self._proxies[key]
@@ -715,21 +955,21 @@
self.read_dict({key: value})
def __delitem__(self, key):
- if key == self._default_section:
+ if key == self.default_section:
raise ValueError("Cannot remove the default section.")
if not self.has_section(key):
raise KeyError(key)
self.remove_section(key)
def __contains__(self, key):
- return key == self._default_section or self.has_section(key)
+ return key == self.default_section or self.has_section(key)
def __len__(self):
return len(self._sections) + 1 # the default section
def __iter__(self):
# XXX does it break when underlying container state changed?
- return itertools.chain((self._default_section,), self._sections.keys())
+ return itertools.chain((self.default_section,), self._sections.keys())
def _read(self, fp, fpname):
"""Parse a sectioned configuration file.
@@ -801,7 +1041,7 @@
lineno)
cursect = self._sections[sectname]
elements_added.add(sectname)
- elif sectname == self._default_section:
+ elif sectname == self.default_section:
cursect = self._defaults
else:
cursect = self._dict()
@@ -836,7 +1076,7 @@
cursect[optname] = [optval]
else:
# valueless option handling
- cursect[optname] = optval
+ cursect[optname] = None
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
@@ -849,12 +1089,16 @@
self._join_multiline_values()
def _join_multiline_values(self):
- all_sections = itertools.chain((self._defaults,),
- self._sections.values())
- for options in all_sections:
+ defaults = self.default_section, self._defaults
+ all_sections = itertools.chain((defaults,),
+ self._sections.items())
+ for section, options in all_sections:
for name, val in options.items():
if isinstance(val, list):
- options[name] = '\n'.join(val).rstrip()
+ val = '\n'.join(val).rstrip()
+ options[name] = self._interpolation.before_read(self,
+ section,
+ name, val)
def _handle_error(self, exc, fpname, lineno, line):
if not exc:
@@ -871,7 +1115,7 @@
try:
d.update(self._sections[section])
except KeyError:
- if section != self._default_section:
+ if section != self.default_section:
raise NoSectionError(section)
# Update with the entry specific variables
if vars:
@@ -906,197 +1150,31 @@
raise TypeError("option values must be strings")
-
class ConfigParser(RawConfigParser):
"""ConfigParser implementing interpolation."""
- def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET):
- """Get an option value for a given section.
+ _DEFAULT_INTERPOLATION = BrokenInterpolation()
- If `vars' is provided, it must be a dictionary. The option is looked up
- in `vars' (if provided), `section', and in `DEFAULTSECT' in that order.
- If the key is not found and `fallback' is provided, it is used as
- a fallback value. `None' can be provided as a `fallback' value.
-
- All % interpolations are expanded in the return values, unless the
- optional argument `raw' is true. Values for interpolation keys are
- looked up in the same manner as the option.
-
- Arguments `raw', `vars', and `fallback' are keyword only.
-
- The section DEFAULT is special.
- """
- try:
- d = self._unify_values(section, vars)
- except NoSectionError:
- if fallback is _UNSET:
- raise
- else:
- return fallback
- option = self.optionxform(option)
- try:
- value = d[option]
- except KeyError:
- if fallback is _UNSET:
- raise NoOptionError(option, section)
- else:
- return fallback
-
- if raw or value is None:
- return value
- else:
- return self._interpolate(section, option, value, d)
-
- def getint(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
- try:
- return self._get(section, int, option, raw=raw, vars=vars)
- except (NoSectionError, NoOptionError):
- if fallback is _UNSET:
- raise
- else:
- return fallback
-
- def getfloat(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
- try:
- return self._get(section, float, option, raw=raw, vars=vars)
- except (NoSectionError, NoOptionError):
- if fallback is _UNSET:
- raise
- else:
- return fallback
-
- def getboolean(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
- try:
- return self._get(section, self._convert_to_boolean, option,
- raw=raw, vars=vars)
- except (NoSectionError, NoOptionError):
- if fallback is _UNSET:
- raise
- else:
- return fallback
-
- def items(self, section, raw=False, vars=None):
- """Return a list of (name, value) tuples for each option in a section.
-
- All % interpolations are expanded in the return values, based on the
- defaults passed into the constructor, unless the optional argument
- `raw' is true. Additional substitutions may be provided using the
- `vars' argument, which must be a dictionary whose contents overrides
- any pre-existing defaults.
-
- The section DEFAULT is special.
- """
- d = self._defaults.copy()
- try:
- d.update(self._sections[section])
- except KeyError:
- if section != self._default_section:
- raise NoSectionError(section)
- # Update with the entry specific variables
- if vars:
- for key, value in vars.items():
- d[self.optionxform(key)] = value
- options = list(d.keys())
- if raw:
- return [(option, d[option])
- for option in options]
- else:
- return [(option, self._interpolate(section, option, d[option], d))
- for option in options]
-
- def _interpolate(self, section, option, rawval, vars):
- # do the string interpolation
- value = rawval
- depth = MAX_INTERPOLATION_DEPTH
- while depth: # Loop through this until it's done
- depth -= 1
- if value and "%(" in value:
- value = self._KEYCRE.sub(self._interpolation_replace, value)
- try:
- value = value % vars
- except KeyError as e:
- raise InterpolationMissingOptionError(
- option, section, rawval, e.args[0])
- else:
- break
- if value and "%(" in value:
- raise InterpolationDepthError(option, section, rawval)
- return value
-
- _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
-
- def _interpolation_replace(self, match):
- s = match.group(1)
- if s is None:
- return match.group()
- else:
- return "%%(%s)s" % self.optionxform(s)
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if self.__class__ is ConfigParser:
+ warnings.warn(
+ "The ConfigParser class will be removed in future versions."
+ " Use SafeConfigParser instead.",
+ DeprecationWarning, stacklevel=2
+ )
class SafeConfigParser(ConfigParser):
"""ConfigParser implementing sane interpolation."""
- def _interpolate(self, section, option, rawval, vars):
- # do the string interpolation
- L = []
- self._interpolate_some(option, L, rawval, section, vars, 1)
- return ''.join(L)
-
- _interpvar_re = re.compile(r"%\(([^)]+)\)s")
-
- def _interpolate_some(self, option, accum, rest, section, map, depth):
- if depth > MAX_INTERPOLATION_DEPTH:
- raise InterpolationDepthError(option, section, rest)
- while rest:
- p = rest.find("%")
- if p < 0:
- accum.append(rest)
- return
- if p > 0:
- accum.append(rest[:p])
- rest = rest[p:]
- # p is no longer used
- c = rest[1:2]
- if c == "%":
- accum.append("%")
- rest = rest[2:]
- elif c == "(":
- m = self._interpvar_re.match(rest)
- if m is None:
- raise InterpolationSyntaxError(option, section,
- "bad interpolation variable reference %r" % rest)
- var = self.optionxform(m.group(1))
- rest = rest[m.end():]
- try:
- v = map[var]
- except KeyError:
- raise InterpolationMissingOptionError(
- option, section, rest, var)
- if "%" in v:
- self._interpolate_some(option, accum, v,
- section, map, depth + 1)
- else:
- accum.append(v)
- else:
- raise InterpolationSyntaxError(
- option, section,
- "'%%' must be followed by '%%' or '(', "
- "found: %r" % (rest,))
+ _DEFAULT_INTERPOLATION = BasicInterpolation()
def set(self, section, option, value=None):
- """Set an option. Extend ConfigParser.set: check for string values."""
+ """Set an option. Extends RawConfigParser.set by validating type and
+ interpolation syntax on the value."""
self._validate_value_type(value)
- # check for bad percent signs
- if value:
- tmp_value = value.replace('%%', '') # escaped percent signs
- tmp_value = self._interpvar_re.sub('', tmp_value) # valid syntax
- if '%' in tmp_value:
- raise ValueError("invalid interpolation syntax in %r at "
- "position %d" % (value, tmp_value.find('%')))
- ConfigParser.set(self, section, option, value)
+ super().set(section, option, value)
class SectionProxy(MutableMapping):