[svn] added baseclasses for loaders, added the memcaching loader, updated documentation for loaders
--HG--
branch : trunk
diff --git a/CHANGES b/CHANGES
index da46f59..29a981d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -112,6 +112,12 @@
- fixed extended slicing
+- reworked loader layer. All the cached loaders now have "private" non cached
+ baseclasses so that you can easily mix your own caching layers in.
+
+- added `MemcachedLoaderMixin` and `MemcachedFileSystemLoader` contributed
+ by Bryan McLemore.
+
Version 1.0
-----------
diff --git a/Makefile b/Makefile
index 9b1cd17..17f0b32 100644
--- a/Makefile
+++ b/Makefile
@@ -25,3 +25,6 @@
release: documentation
@(python2.3 setup.py release bdist_egg upload; python2.4 setup.py release bdist_egg upload; python2.5 setup.py release bdist_egg sdist upload)
+
+test-release: documentation
+ @(python2.3 setup.py release bdist_egg; python2.4 setup.py release bdist_egg; python2.5 setup.py release bdist_egg sdist)
diff --git a/docs/generate.py b/docs/generate.py
index f8e8238..6b6b0ac 100755
--- a/docs/generate.py
+++ b/docs/generate.py
@@ -101,6 +101,30 @@
return '\n\n'.join(result)
+def generate_list_of_baseloaders():
+ from jinja import loaders as loader_module
+
+ result = []
+ loaders = []
+ for item in dir(loader_module):
+ obj = getattr(loader_module, item)
+ try:
+ if issubclass(obj, loader_module.BaseLoader) and \
+ obj.__name__ != 'BaseLoader' and \
+ obj.__name__ not in loader_module.__all__:
+ loaders.append(obj)
+ except TypeError:
+ pass
+ loaders.sort(key=lambda x: x.__name__.lower())
+
+ for loader in loaders:
+ doclines = []
+ for line in inspect.getdoc(loader).splitlines():
+ doclines.append(' ' + line)
+ result.append('`%s`\n%s' % (loader.__name__, '\n'.join(doclines)))
+
+ return '\n\n'.join(result)
+
def generate_environment_doc():
from jinja.environment import Environment
return '%s\n\n%s' % (
@@ -115,6 +139,7 @@
LIST_OF_FILTERS = generate_list_of_filters()
LIST_OF_TESTS = generate_list_of_tests()
LIST_OF_LOADERS = generate_list_of_loaders()
+LIST_OF_BASELOADERS = generate_list_of_baseloaders()
ENVIRONMENT_DOC = generate_environment_doc()
CHANGELOG = file(os.path.join(os.path.dirname(__file__), os.pardir, 'CHANGES'))\
.read().decode('utf-8')
@@ -240,6 +265,7 @@
data = data.replace('[[list_of_filters]]', LIST_OF_FILTERS)\
.replace('[[list_of_tests]]', LIST_OF_TESTS)\
.replace('[[list_of_loaders]]', LIST_OF_LOADERS)\
+ .replace('[[list_of_baseloaders]]', LIST_OF_BASELOADERS)\
.replace('[[environment_doc]]', ENVIRONMENT_DOC)\
.replace('[[changelog]]', CHANGELOG)
parts = publish_parts(
diff --git a/docs/src/loaders.txt b/docs/src/loaders.txt
index 29da849..306b14d 100644
--- a/docs/src/loaders.txt
+++ b/docs/src/loaders.txt
@@ -7,8 +7,21 @@
Builtin Loaders
===============
+This list contains the builtin loaders you can use without further
+modification:
+
[[list_of_loaders]]
+Loader Baseclasses
+==================
+
+With Jinja 1.1 onwards all the loaders have (except of the uncached) now have
+baseclasses you can use to mix your own caching layer in. This technique is
+explained below. The `BaseLoader` itself is also a loader baseclass but because
+it's the baseclass of each loader it's explained in detail below.
+
+[[list_of_baseloaders]]
+
Developing Loaders
==================
@@ -43,7 +56,13 @@
f.close()
The functions `load` and `parse` which are a requirement for a loader are
-added automatically by the `BaseLoader`.
+added automatically by the `BaseLoader`. Instead of the normal `BaseLoader`
+you can use one of the other base loaders that already come with a proper
+`get_source` method for further modification. Those loaders however are
+new in Jinja 1.1.
+
+CachedLoaderMixin
+-----------------
Additionally to the `BaseLoader` there is a mixin class called
`CachedLoaderMixin` that implements memory and disk caching of templates.
@@ -91,4 +110,37 @@
exist the option `auto_reload` won't have an effect. Also note that the
`check_source_changed` method must not raise an exception if the template
does not exist but return ``-1``. The return value ``-1`` is considered
-"always reload" whereas ``0`` means "do not reload".
+"always reload" whereas ``0`` means "do not reload". The default return
+value for not existing templates should be ``-1``.
+
+For the default base classes that come with Jinja 1.1 onwards there exist
+also concrete implementations that support caching. The implementation
+just mixes in the `CachedLoaderMixin`.
+
+MemcachedLoaderMixin
+--------------------
+
+*New in Jinja 1.1*
+
+The `MemcachedLoaderMixin` class adds support for `memcached`_ caching.
+There is only one builtin loader that mixes it in: The
+`MemcachedFileSystemLoader`. If you need other loaders with this mixin
+you can easily subclass one of the existing base loaders. Here an example
+for the `FunctionLoader`:
+
+.. sourcecode:: python
+
+ from jinja.loaders import FunctionLoader, MemcachedLoaderMixin
+
+ class MemcachedFunctionLoader(MemcachedLoaderMixin, FunctionLoader):
+
+ def __init__(self, loader_func):
+ BaseFunctionLoader.__init__(self, loader_func)
+ MemcachedLoaderMixin.__init__(self,
+ True, # use memcached
+ 60 * 60 * 24 * 7, # 7 days expiration
+ ['127.0.0.1:11211'], # the memcached hosts
+ 'template/' # string prefix for the cache keys
+ )
+
+.. _memcached: http://www.danga.com/memcached/
diff --git a/jinja/loaders.py b/jinja/loaders.py
index 27999a2..c6b82f0 100644
--- a/jinja/loaders.py
+++ b/jinja/loaders.py
@@ -5,7 +5,7 @@
Jinja loader classes.
- :copyright: 2007 by Armin Ronacher.
+ :copyright: 2007 by Armin Ronacher, Bryan McLemore.
:license: BSD, see LICENSE for more details.
"""
@@ -22,7 +22,7 @@
#: when updating this, update the listing in the jinja package too
__all__ = ['FileSystemLoader', 'PackageLoader', 'DictLoader', 'ChoiceLoader',
- 'FunctionLoader']
+ 'FunctionLoader', 'MemcachedFileSystemLoader']
def get_template_filename(searchpath, name):
@@ -142,7 +142,14 @@
class CachedLoaderMixin(object):
"""
- Mixin this class to implement simple memory and disk caching.
+ Mixin this class to implement simple memory and disk caching. The
+ memcaching just uses a dict in the loader so if you have a global
+ environment or at least a global loader this can speed things up.
+
+ If the memcaching is enabled you can use (with Jinja 1.1 onwards)
+ the `clear_memcache` function to clear the cache.
+
+ For memcached support check the `MemcachedLoaderMixin`.
"""
def __init__(self, use_memcache, cache_size, cache_folder, auto_reload,
@@ -160,13 +167,20 @@
self.__times = {}
self.__lock = Lock()
+ def clear_memcache(self):
+ """
+ Clears the memcache.
+ """
+ if self.__memcache is not None:
+ self.__memcache.clear()
+
def load(self, environment, name, translator):
"""
Load and translate a template. First we check if there is a
cached version of this template in the memory cache. If this is
not the cache check for a compiled template in the disk cache
- folder. And if none of this is the case we translate the temlate
- using the `LoaderMixin.load` function, cache and return it.
+ folder. And if none of this is the case we translate the temlate,
+ cache and return it.
"""
self.__lock.acquire()
try:
@@ -193,8 +207,7 @@
if name in self.__memcache:
tmpl = self.__memcache[name]
# if auto reload is enabled check if the template changed
- if last_change is not None and \
- last_change > self.__times[name]:
+ if last_change and last_change > self.__times[name]:
tmpl = None
push_to_memory = True
else:
@@ -247,7 +260,105 @@
self.__lock.release()
-class FileSystemLoader(CachedLoaderMixin, BaseLoader):
+class MemcachedLoaderMixin(object):
+ """
+ Uses a memcached server to cache the templates.
+ """
+
+ def __init__(self, use_memcache, memcache_time=60 * 60 * 24 * 7,
+ memcache_host=None, item_prefix='template/'):
+ try:
+ from memcache import Client
+ except ImportError:
+ raise RuntimeError('the %r loader requires an installed '
+ 'memcache module' % self.__class__.__name__)
+ if memcache_host is None:
+ memcache_host = ['127.0.0.1:11211']
+ if use_memcache:
+ self.__memcache = Client(list(memcache_host))
+ self.__memcache_time = memcache_time
+ else:
+ self.__memcache = None
+ self.__item_prefix = item_prefix
+ self.__lock = Lock()
+
+ def load(self, environment, name, translator):
+ """
+ Load and translate a template. First we check if there is a
+ cached version of this template in the memory cache. If this is
+ not the cache check for a compiled template in the disk cache
+ folder. And if none of this is the case we translate the template,
+ cache and return it.
+ """
+ self.__lock.acquire()
+ try:
+ # caching is only possible for the python translator. skip
+ # all other translators
+ if translator is not PythonTranslator:
+ return super(MemcachedLoaderMixin, self).load(
+ environment, name, translator)
+ tmpl = None
+ push_to_memory = False
+
+ # check if we have something in the memory cache and the
+ # memory cache is enabled.
+ if self.__memcache is not None:
+ bytecode = self.__memcache.get(self.__item_prefix + name)
+ if bytecode:
+ tmpl = Template.load(environment, bytecode)
+ else:
+ push_to_memory = True
+
+ # if we still have no template we load, parse and translate it.
+ if tmpl is None:
+ tmpl = super(MemcachedLoaderMixin, self).load(
+ environment, name, translator)
+
+ # if memcaching is enabled and the template not loaded
+ # we add that there.
+ if push_to_memory:
+ self.__memcache.set(self.__item_prefix + name, tmpl.dump(),
+ self.__memcache_time)
+ return tmpl
+ finally:
+ self.__lock.release()
+
+
+class BaseFileSystemLoader(BaseLoader):
+ """
+ Baseclass for the file system loader that does not do any caching.
+ It exists to avoid redundant code, just don't use it without subclassing.
+
+ How subclassing can work:
+
+ .. sourcecode:: python
+
+ from jinja.loaders import BaseFileSystemLoader
+
+ class MyFileSystemLoader(BaseFileSystemLoader):
+ def __init__(self):
+ BaseFileSystemLoader.__init__(self, '/path/to/templates')
+
+ The base file system loader only takes one parameter beside self which
+ is the path to the templates.
+ """
+
+ def __init__(self, searchpath):
+ self.searchpath = path.abspath(searchpath)
+
+ def get_source(self, environment, name, parent):
+ filename = get_template_filename(self.searchpath, name)
+ if path.exists(filename):
+ f = codecs.open(filename, 'r', environment.template_charset)
+ try:
+ return f.read()
+ finally:
+ f.close()
+ else:
+ raise TemplateNotFound(name)
+
+
+class FileSystemLoader(CachedLoaderMixin, BaseFileSystemLoader):
"""
Loads templates from the filesystem:
@@ -257,7 +368,7 @@
e = Environment(loader=FileSystemLoader('templates/'))
You can pass the following keyword arguments to the loader on
- initialisation:
+ initialization:
=================== =================================================
``searchpath`` String with the path to the templates on the
@@ -287,23 +398,13 @@
def __init__(self, searchpath, use_memcache=False, memcache_size=40,
cache_folder=None, auto_reload=True, cache_salt=None):
- self.searchpath = path.abspath(searchpath)
+ BaseFileSystemLoader.__init__(self, searchpath)
+
if cache_salt is None:
cache_salt = self.searchpath
CachedLoaderMixin.__init__(self, use_memcache, memcache_size,
cache_folder, auto_reload, cache_salt)
- def get_source(self, environment, name, parent):
- filename = get_template_filename(self.searchpath, name)
- if path.exists(filename):
- f = codecs.open(filename, 'r', environment.template_charset)
- try:
- return f.read()
- finally:
- f.close()
- else:
- raise TemplateNotFound(name)
-
def check_source_changed(self, environment, name):
filename = get_template_filename(self.searchpath, name)
if path.exists(filename):
@@ -311,7 +412,84 @@
return -1
-class PackageLoader(CachedLoaderMixin, BaseLoader):
+class MemcachedFileSystemLoader(MemcachedLoaderMixin, BaseFileSystemLoader):
+ """
+ Loads templates from the filesystem and caches them on a memcached
+ server.
+
+ .. sourcecode:: python
+
+ from jinja import Environment, MemcachedFileSystemLoader
+ e = Environment(loader=MemcachedFileSystemLoader('templates/',
+ memcache_host=['192.168.2.250:11211']
+ ))
+
+ You can pass the following keyword arguments to the loader on
+ initialization:
+
+ =================== =================================================
+ ``searchpath`` String with the path to the templates on the
+ filesystem.
+ ``use_memcache`` Set this to ``True`` to enable memcached caching.
+ In that case it behaves like a normal
+ `FileSystemLoader` with disabled caching.
+ ``memcache_time`` The expire time of a template in the cache.
+ ``memcache_host`` a list of memcached servers.
+ ``item_prefix`` The prefix for the items on the server. Defaults
+ to ``'template/'``.
+ =================== =================================================
+ """
+
+ def __init__(self, searchpath, use_memcache=True,
+ memcache_time=60 * 60 * 24 * 7, memcache_host=None,
+ item_prefix='template/'):
+ BaseFileSystemLoader.__init__(self, searchpath)
+ MemcachedLoaderMixin.__init__(self, use_memcache, memcache_time,
+ memcache_host, item_prefix)
+
+
+class BasePackageLoader(BaseLoader):
+ """
+ Baseclass for the package loader that does not do any caching.
+
+ It accepts two parameters: The name of the package and the path relative
+ to the package:
+
+ .. sourcecode:: python
+
+ from jinja.loaders import BasePackageLoader
+
+ class MyPackageLoader(BasePackageLoader):
+ def __init__(self):
+ BasePackageLoader.__init__(self, 'my_package', 'shared/templates')
+
+ The relative path must use slashes as path delimiters, even on Mac OS
+ and Microsoft Windows.
+
+ It uses the `pkg_resources` libraries distributed with setuptools for
+ retrieving the data from the packages. This works for eggs too so you
+ don't have to mark your egg as non zip safe.
+ """
+
+ def __init__(self, package_name, package_path):
+ try:
+ import pkg_resources
+ except ImportError:
+ raise RuntimeError('setuptools not installed')
+ self.package_name = package_name
+ self.package_path = package_path
+
+ def get_source(self, environment, name, parent):
+ from pkg_resources import resource_exists, resource_string
+ path = '/'.join([self.package_path] + [p for p in name.split('/')
+ if p != '..'])
+ if not resource_exists(self.package_name, path):
+ raise TemplateNotFound(name)
+ contents = resource_string(self.package_name, path)
+ return contents.decode(environment.template_charset)
+
+
+class PackageLoader(CachedLoaderMixin, BasePackageLoader):
"""
Loads templates from python packages using setuptools.
@@ -321,7 +499,7 @@
e = Environment(loader=PackageLoader('yourapp', 'template/path'))
You can pass the following keyword arguments to the loader on
- initialisation:
+ initialization:
=================== =================================================
``package_name`` Name of the package containing the templates.
@@ -361,26 +539,13 @@
def __init__(self, package_name, package_path, use_memcache=False,
memcache_size=40, cache_folder=None, auto_reload=True,
cache_salt=None):
- try:
- import pkg_resources
- except ImportError:
- raise RuntimeError('setuptools not installed')
- self.package_name = package_name
- self.package_path = package_path
+ BasePackageLoader.__init__(self, package_name, package_path)
+
if cache_salt is None:
cache_salt = package_name + '/' + package_path
CachedLoaderMixin.__init__(self, use_memcache, memcache_size,
cache_folder, auto_reload, cache_salt)
- def get_source(self, environment, name, parent):
- from pkg_resources import resource_exists, resource_string
- path = '/'.join([self.package_path] + [p for p in name.split('/')
- if p != '..'])
- if not resource_exists(self.package_name, path):
- raise TemplateNotFound(name)
- contents = resource_string(self.package_name, path)
- return contents.decode(environment.template_charset)
-
def check_source_changed(self, environment, name):
from pkg_resources import resource_exists, resource_filename
fn = resource_filename(self.package_name, '/'.join([self.package_path] +
@@ -390,7 +555,38 @@
return -1
-class FunctionLoader(CachedLoaderMixin, BaseLoader):
+class BaseFunctionLoader(BaseLoader):
+ """
+ Baseclass for the function loader that doesn't do any caching.
+
+ It just accepts one parameter which is the function which is called
+ with the name of the requested template. If the return value is `None`
+ the loader will raise a `TemplateNotFound` error.
+
+ .. sourcecode:: python
+
+ from jinja.loaders import BaseFunctionLoader
+
+ templates = {...}
+
+ class MyFunctionLoader(BaseFunctionLoader):
+ def __init__(self):
+ BaseFunctionLoader(templates.get)
+ """
+
+ def __init__(self, loader_func):
+ self.loader_func = loader_func
+
+ def get_source(self, environment, name, parent):
+ rv = self.loader_func(name)
+ if rv is None:
+ raise TemplateNotFound(name)
+ if isinstance(rv, str):
+ return rv.decode(environment.template_charset)
+ return rv
+
+
+class FunctionLoader(CachedLoaderMixin, BaseFunctionLoader):
"""
Loads templates by calling a function which has to return a string
or `None` if an error occoured.
@@ -410,7 +606,7 @@
solid backend.
You can pass the following keyword arguments to the loader on
- initialisation:
+ initialization:
=================== =================================================
``loader_func`` Function that takes the name of the template to
@@ -446,23 +642,15 @@
def __init__(self, loader_func, getmtime_func=None, use_memcache=False,
memcache_size=40, cache_folder=None, auto_reload=True,
cache_salt=None):
+ BaseFunctionLoader.__init__(self, loader_func)
# when changing the signature also check the jinja.plugin function
# loader instantiation.
- self.loader_func = loader_func
self.getmtime_func = getmtime_func
if auto_reload and getmtime_func is None:
auto_reload = False
CachedLoaderMixin.__init__(self, use_memcache, memcache_size,
cache_folder, auto_reload, cache_salt)
- def get_source(self, environment, name, parent):
- rv = self.loader_func(name)
- if rv is None:
- raise TemplateNotFound(name)
- if isinstance(rv, str):
- return rv.decode(environment.template_charset)
- return rv
-
def check_source_changed(self, environment, name):
return self.getmtime_func(name)
diff --git a/jinja/utils.py b/jinja/utils.py
index d9608eb..42eec0d 100644
--- a/jinja/utils.py
+++ b/jinja/utils.py
@@ -71,7 +71,6 @@
#: function types
callable_types = (FunctionType, MethodType)
-
#: number of maximal range items
MAX_RANGE = 1000000
diff --git a/tests/test_syntax.py b/tests/test_syntax.py
index 79aaf2f..e6a8714 100644
--- a/tests/test_syntax.py
+++ b/tests/test_syntax.py
@@ -19,6 +19,7 @@
COMPARE = '''{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|{{ 2 == 2 }}|{{ 1 <= 1 }}'''
LITERALS = '''{{ [] }}|{{ {} }}|{{ '' }}'''
BOOL = '''{{ true and false }}|{{ false or true }}|{{ not false }}'''
+GROUPING = '''{{ (true and false) or (false and true) and not false }}'''
def test_call():
@@ -82,3 +83,8 @@
def test_bool(env):
tmpl = env.from_string(BOOL)
assert tmpl.render() == 'False|True|True'
+
+
+def test_grouping(env):
+ tmpl = env.from_string(GROUPING)
+ assert tmpl.render() == 'False'