blob: 9bebb11ac4b542e467c907ce09b8e687175307b5 [file] [log] [blame]
Barry Warsaw95be23d2000-08-25 19:13:37 +00001"""Internationalization and localization support.
2
3This module provides internationalization (I18N) and localization (L10N)
4support for your Python programs by providing an interface to the GNU gettext
5message catalog library.
6
7I18N refers to the operation by which a program is made aware of multiple
8languages. L10N refers to the adaptation of your program, once
Barry Warsaw33d8d702000-08-30 03:29:58 +00009internationalized, to the local language and cultural habits.
Barry Warsaw95be23d2000-08-25 19:13:37 +000010
11"""
12
Barry Warsawfa488ec2000-08-25 20:26:43 +000013# This module represents the integration of work, contributions, feedback, and
14# suggestions from the following people:
Barry Warsaw95be23d2000-08-25 19:13:37 +000015#
16# Martin von Loewis, who wrote the initial implementation of the underlying
17# C-based libintlmodule (later renamed _gettext), along with a skeletal
18# gettext.py implementation.
19#
20# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21# which also included a pure-Python implementation to read .mo files if
22# intlmodule wasn't available.
23#
24# James Henstridge, who also wrote a gettext.py module, which has some
25# interesting, but currently unsupported experimental features: the notion of
26# a Catalog class and instances, and the ability to add to a catalog file via
27# a Python API.
28#
29# Barry Warsaw integrated these modules, wrote the .install() API and code,
30# and conformed all C and Python code to Python's coding standards.
Barry Warsaw33d8d702000-08-30 03:29:58 +000031#
32# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
33# module.
34#
35# TODO:
36# - Lazy loading of .mo files. Currently the entire catalog is loaded into
37# memory, but that's probably bad for large translated programs. Instead,
38# the lexical sort of original strings in GNU .mo files should be exploited
39# to do binary searches and lazy initializations. Or you might want to use
40# the undocumented double-hash algorithm for .mo files with hash tables, but
41# you'll need to study the GNU gettext code to do this.
42#
43# - Support Solaris .mo file formats. Unfortunately, we've been unable to
44# find this format documented anywhere.
Barry Warsaw95be23d2000-08-25 19:13:37 +000045
46import os
47import sys
48import struct
Barry Warsaw33d8d702000-08-30 03:29:58 +000049from errno import ENOENT
Barry Warsaw95be23d2000-08-25 19:13:37 +000050
Barry Warsaw33d8d702000-08-30 03:29:58 +000051_default_localedir = os.path.join(sys.prefix, 'share', 'locale')
Barry Warsaw95be23d2000-08-25 19:13:37 +000052
53
54
Barry Warsawfa488ec2000-08-25 20:26:43 +000055def _expand_lang(locale):
56 from locale import normalize
57 locale = normalize(locale)
58 COMPONENT_CODESET = 1 << 0
59 COMPONENT_TERRITORY = 1 << 1
60 COMPONENT_MODIFIER = 1 << 2
61 # split up the locale into its base components
62 mask = 0
63 pos = locale.find('@')
64 if pos >= 0:
65 modifier = locale[pos:]
66 locale = locale[:pos]
67 mask |= COMPONENT_MODIFIER
68 else:
69 modifier = ''
70 pos = locale.find('.')
71 if pos >= 0:
72 codeset = locale[pos:]
73 locale = locale[:pos]
74 mask |= COMPONENT_CODESET
75 else:
76 codeset = ''
77 pos = locale.find('_')
78 if pos >= 0:
79 territory = locale[pos:]
80 locale = locale[:pos]
81 mask |= COMPONENT_TERRITORY
82 else:
83 territory = ''
84 language = locale
85 ret = []
86 for i in range(mask+1):
87 if not (i & ~mask): # if all components for this combo exist ...
88 val = language
89 if i & COMPONENT_TERRITORY: val += territory
90 if i & COMPONENT_CODESET: val += codeset
91 if i & COMPONENT_MODIFIER: val += modifier
92 ret.append(val)
93 ret.reverse()
94 return ret
95
96
97
Barry Warsaw33d8d702000-08-30 03:29:58 +000098class NullTranslations:
99 def __init__(self, fp=None):
100 self._info = {}
101 self._charset = None
102 if fp:
103 self._parse(fp)
Barry Warsaw95be23d2000-08-25 19:13:37 +0000104
Barry Warsaw33d8d702000-08-30 03:29:58 +0000105 def _parse(self, fp):
106 pass
107
108 def gettext(self, message):
109 return message
110
111 def ugettext(self, message):
112 return unicode(message)
113
114 def info(self):
115 return self._info
116
117 def charset(self):
118 return self._charset
119
120 def install(self, unicode=0):
121 import __builtin__
122 __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
123
124
125class GNUTranslations(NullTranslations):
126 # Magic number of .mo files
127 LE_MAGIC = 0x950412de
Barry Warsawb76a55c2000-08-31 10:45:54 +0000128 BE_MAGIC = 0xde120495
Barry Warsaw95be23d2000-08-25 19:13:37 +0000129
130 def _parse(self, fp):
131 """Override this method to support alternative .mo formats."""
Tim Petersc6387912000-09-01 02:20:20 +0000132 # We need to & all 32 bit unsigned integers with 0xffffffff for
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000133 # portability to 64 bit machines.
134 MASK = 0xffffffff
Barry Warsaw95be23d2000-08-25 19:13:37 +0000135 unpack = struct.unpack
136 filename = getattr(fp, 'name', '')
137 # Parse the .mo file header, which consists of 5 little endian 32
138 # bit words.
Barry Warsaw33d8d702000-08-30 03:29:58 +0000139 self._catalog = catalog = {}
Barry Warsaw95be23d2000-08-25 19:13:37 +0000140 buf = fp.read()
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000141 buflen = len(buf)
Barry Warsaw33d8d702000-08-30 03:29:58 +0000142 # Are we big endian or little endian?
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000143 magic = unpack('<i', buf[:4])[0] & MASK
Barry Warsaw33d8d702000-08-30 03:29:58 +0000144 if magic == self.LE_MAGIC:
145 version, msgcount, masteridx, transidx = unpack('<4i', buf[4:20])
146 ii = '<ii'
147 elif magic == self.BE_MAGIC:
148 version, msgcount, masteridx, transidx = unpack('>4i', buf[4:20])
149 ii = '>ii'
150 else:
Barry Warsaw95be23d2000-08-25 19:13:37 +0000151 raise IOError(0, 'Bad magic number', filename)
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000152 # more unsigned ints
153 msgcount &= MASK
154 masteridx &= MASK
155 transidx &= MASK
Barry Warsaw95be23d2000-08-25 19:13:37 +0000156 # Now put all messages from the .mo file buffer into the catalog
157 # dictionary.
158 for i in xrange(0, msgcount):
Barry Warsaw33d8d702000-08-30 03:29:58 +0000159 mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000160 moff &= MASK
161 mend = moff + (mlen & MASK)
Barry Warsaw33d8d702000-08-30 03:29:58 +0000162 tlen, toff = unpack(ii, buf[transidx:transidx+8])
Barry Warsaw9a2d9d72000-08-31 23:28:52 +0000163 toff &= MASK
164 tend = toff + (tlen & MASK)
165 if mend < buflen and tend < buflen:
Barry Warsaw33d8d702000-08-30 03:29:58 +0000166 tmsg = buf[toff:tend]
167 catalog[buf[moff:mend]] = tmsg
Barry Warsaw95be23d2000-08-25 19:13:37 +0000168 else:
169 raise IOError(0, 'File is corrupt', filename)
Barry Warsaw33d8d702000-08-30 03:29:58 +0000170 # See if we're looking at GNU .mo conventions for metadata
171 if mlen == 0 and tmsg.lower().startswith('project-id-version:'):
172 # Catalog description
173 for item in tmsg.split('\n'):
174 item = item.strip()
175 if not item:
176 continue
177 k, v = item.split(':', 1)
178 k = k.strip().lower()
179 v = v.strip()
180 self._info[k] = v
181 if k == 'content-type':
182 self._charset = v.split('charset=')[1]
Barry Warsaw95be23d2000-08-25 19:13:37 +0000183 # advance to next entry in the seek tables
Barry Warsawfa488ec2000-08-25 20:26:43 +0000184 masteridx += 8
185 transidx += 8
Barry Warsaw33d8d702000-08-30 03:29:58 +0000186
187 def gettext(self, message):
188 return self._catalog.get(message, message)
189
190 def ugettext(self, message):
191 tmsg = self._catalog.get(message, message)
192 return unicode(tmsg, self._charset)
Barry Warsaw95be23d2000-08-25 19:13:37 +0000193
194
195
Barry Warsaw95be23d2000-08-25 19:13:37 +0000196# Locate a .mo file using the gettext strategy
Barry Warsaw33d8d702000-08-30 03:29:58 +0000197def find(domain, localedir=None, languages=None):
Barry Warsaw95be23d2000-08-25 19:13:37 +0000198 # Get some reasonable defaults for arguments that were not supplied
Barry Warsaw95be23d2000-08-25 19:13:37 +0000199 if localedir is None:
Barry Warsaw33d8d702000-08-30 03:29:58 +0000200 localedir = _default_localedir
Barry Warsaw95be23d2000-08-25 19:13:37 +0000201 if languages is None:
202 languages = []
203 for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
204 val = os.environ.get(envar)
205 if val:
206 languages = val.split(':')
207 break
208 if 'C' not in languages:
209 languages.append('C')
Barry Warsawfa488ec2000-08-25 20:26:43 +0000210 # now normalize and expand the languages
Barry Warsaw75f81012000-10-16 15:47:50 +0000211 nelangs = []
Barry Warsawfa488ec2000-08-25 20:26:43 +0000212 for lang in languages:
213 for nelang in _expand_lang(lang):
Barry Warsaw75f81012000-10-16 15:47:50 +0000214 if nelang not in nelangs:
215 nelangs.append(nelang)
Barry Warsaw95be23d2000-08-25 19:13:37 +0000216 # select a language
Barry Warsaw75f81012000-10-16 15:47:50 +0000217 for lang in nelangs:
Barry Warsaw95be23d2000-08-25 19:13:37 +0000218 if lang == 'C':
219 break
Barry Warsaw84314b72000-08-25 19:53:17 +0000220 mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
Barry Warsaw33d8d702000-08-30 03:29:58 +0000221 if os.path.exists(mofile):
222 return mofile
223 return None
Barry Warsaw95be23d2000-08-25 19:13:37 +0000224
225
226
Barry Warsaw33d8d702000-08-30 03:29:58 +0000227# a mapping between absolute .mo file path and Translation object
228_translations = {}
229
230def translation(domain, localedir=None, languages=None, class_=None):
231 if class_ is None:
232 class_ = GNUTranslations
233 mofile = find(domain, localedir, languages)
234 if mofile is None:
235 raise IOError(ENOENT, 'No translation file found for domain', domain)
236 key = os.path.abspath(mofile)
237 # TBD: do we need to worry about the file pointer getting collected?
Barry Warsaw293b03f2000-10-05 18:48:12 +0000238 # Avoid opening, reading, and parsing the .mo file after it's been done
239 # once.
240 t = _translations.get(key)
241 if t is None:
242 t = _translations.setdefault(key, class_(open(mofile, 'rb')))
Barry Warsaw33d8d702000-08-30 03:29:58 +0000243 return t
244
245
246
247def install(domain, localedir=None, unicode=0):
248 translation(domain, localedir).install(unicode)
249
250
251
252# a mapping b/w domains and locale directories
253_localedirs = {}
254# current global domain, `messages' used for compatibility w/ GNU gettext
255_current_domain = 'messages'
Barry Warsaw95be23d2000-08-25 19:13:37 +0000256
257
258def textdomain(domain=None):
Barry Warsaw95be23d2000-08-25 19:13:37 +0000259 global _current_domain
Barry Warsaw33d8d702000-08-30 03:29:58 +0000260 if domain is not None:
Barry Warsaw95be23d2000-08-25 19:13:37 +0000261 _current_domain = domain
Barry Warsaw33d8d702000-08-30 03:29:58 +0000262 return _current_domain
Barry Warsaw95be23d2000-08-25 19:13:37 +0000263
264
Barry Warsaw33d8d702000-08-30 03:29:58 +0000265def bindtextdomain(domain, localedir=None):
266 global _localedirs
267 if localedir is not None:
268 _localedirs[domain] = localedir
269 return _localedirs.get(domain, _default_localedir)
Barry Warsaw95be23d2000-08-25 19:13:37 +0000270
271
272def dgettext(domain, message):
Barry Warsaw33d8d702000-08-30 03:29:58 +0000273 try:
274 t = translation(domain, _localedirs.get(domain, None))
275 except IOError:
276 return message
277 return t.gettext(message)
278
279
280def gettext(message):
281 return dgettext(_current_domain, message)
Barry Warsaw95be23d2000-08-25 19:13:37 +0000282
283
Barry Warsaw33d8d702000-08-30 03:29:58 +0000284# dcgettext() has been deemed unnecessary and is not implemented.
Barry Warsaw95be23d2000-08-25 19:13:37 +0000285
Barry Warsaw33d8d702000-08-30 03:29:58 +0000286# James Henstridge's Catalog constructor from GNOME gettext. Documented usage
287# was:
288#
289# import gettext
290# cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
291# _ = cat.gettext
292# print _('Hello World')
Barry Warsaw95be23d2000-08-25 19:13:37 +0000293
Barry Warsaw33d8d702000-08-30 03:29:58 +0000294# The resulting catalog object currently don't support access through a
295# dictionary API, which was supported (but apparently unused) in GNOME
296# gettext.
Barry Warsaw95be23d2000-08-25 19:13:37 +0000297
Barry Warsaw33d8d702000-08-30 03:29:58 +0000298Catalog = translation