Initial implementation
diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py
index ee3ecd2..3542f3f 100755
--- a/Tools/i18n/pygettext.py
+++ b/Tools/i18n/pygettext.py
@@ -1 +1,269 @@
#! /usr/bin/env python
+
+"""pygettext -- Python equivalent of xgettext(1)
+
+Many systems (Solaris, Linux, Gnu) provide extensive tools that ease the
+internationalization of C programs. Most of these tools are independent of
+the programming language and can be used from within Python programs. Martin
+von Loewis' work[1] helps considerably in this regard.
+
+There's one hole though; xgettext is the program that scans source code
+looking for message strings, but it groks only C (or C++). Python introduces
+a few wrinkles, such as dual quoting characters, triple quoted strings, and
+raw strings. xgettext understands none of this.
+
+Enter pygettext, which uses Python's standard tokenize module to scan Python
+source code, generating .pot files identical to what GNU xgettext[2] generates
+for C and C++ code. From there, the standard GNU tools can be used.
+
+A word about marking Python strings as candidates for translation. GNU
+xgettext recognizes the following keywords: gettext, dgettext, dcgettext, and
+gettext_noop. But those can be a lot of text to include all over your code.
+C and C++ have a trick: they use the C preprocessor. Most internationalized C
+source includes a #define for gettext() to _() so that what has to be written
+in the source is much less. Thus these are both translatable strings:
+
+ gettext("Translatable String")
+ _("Translatable String")
+
+Python of course has no preprocessor so this doesn't work so well. Thus,
+pygettext searches only for _() by default, but see the -k/--keyword flag
+below for how to augment this.
+
+ [1] http://www.python.org/workshops/1997-10/proceedings/loewis.html
+ [2] http://www.gnu.org/software/gettext/gettext.html
+
+
+NOTE: pygettext attempts to be option and feature compatible with GNU xgettext
+where ever possible.
+
+Usage: pygettext [options] filename ...
+
+Options:
+
+ -a
+ --extract-all
+ Extract all strings
+
+ -d default-domain
+ --default-domain=default-domain
+ Rename the default output file from messages.pot to default-domain.pot
+
+ -k [word]
+ --keyword[=word]
+ Additional keywords to look for. Without `word' means not to use the
+ default keywords. The default keywords, which are always looked for
+ if not explicitly disabled: _
+
+ The default keyword list is different than GNU xgettext. You can have
+ multiple -k flags on the command line.
+
+ --no-location
+ Do not write filename/lineno location comments
+
+ -n [style]
+ --add-location[=style]
+ Write filename/lineno location comments indicating where each
+ extracted string is found in the source. These lines appear before
+ each msgid. Two styles are supported:
+
+ Solaris # File: filename, line: line-number
+ Gnu #: filename:line
+
+ If style is omitted, Gnu is used. The style name is case
+ insensitive. By default, locations are included.
+
+ --help
+ -h
+ print this help message and exit
+
+"""
+
+import os
+import sys
+import string
+import time
+import getopt
+import tokenize
+
+__version__ = '0.1'
+
+
+
+def usage(code, msg=''):
+ print __doc__ % globals()
+ if msg:
+ print msg
+ sys.exit(code)
+
+
+
+def normalize(s):
+ # This converts the various Python string types into a format that is
+ # appropriate for .po files, namely much closer to C style.
+ #
+ # unwrap quotes, safely
+ s = eval(s, {'__builtins__':{}}, {})
+ # now escape any embedded double quotes
+ parts = []
+ last = 0
+ i = string.find(s, '"')
+ while i >= 0:
+ # find the number of preceding backslashes
+ j = i
+ n = 0
+ while j >= 0 and s[i] == '\\':
+ j = j - 1
+ n = n + 1
+ if (n % 2) == 0:
+ parts.append(s[last:j])
+ parts.append('\\')
+ parts.append(s[j:i])
+ else:
+ parts.append(s[last:i])
+ last = i
+ i = string.find(s, '"', i+1)
+ else:
+ parts.append(s[last:])
+ if parts:
+ return '"' + string.join(parts, '') + '"'
+ else:
+ return '"' + s + '"'
+
+
+
+class TokenEater:
+ def __init__(self, options):
+ self.__options = options
+ self.__messages = {}
+ self.__state = self.__waiting
+ self.__data = []
+ self.__lineno = -1
+
+ def __call__(self, ttype, tstring, stup, etup, line):
+ # dispatch
+ self.__state(ttype, tstring, stup[0])
+
+ def __waiting(self, ttype, tstring, lineno):
+ if ttype == tokenize.NAME and tstring in self.__options.keywords:
+ self.__state = self.__keywordseen
+
+ def __keywordseen(self, ttype, tstring, lineno):
+ if ttype == tokenize.OP and tstring == '(':
+ self.__data = []
+ self.__lineno = lineno
+ self.__state = self.__openseen
+ else:
+ self.__state = self.__waiting
+
+ def __openseen(self, ttype, tstring, lineno):
+ if ttype == tokenize.OP and tstring == ')':
+ # We've seen the last of the translatable strings. Record the
+ # line number of the first line of the strings and update the list
+ # of messages seen. Reset state for the next batch. If there
+ # were no strings inside _(), then just ignore this entry.
+ if self.__data:
+ msg = string.join(self.__data, '')
+ entry = (self.__curfile, self.__lineno)
+ linenos = self.__messages.get(msg)
+ if linenos is None:
+ self.__messages[msg] = [entry]
+ else:
+ linenos.append(entry)
+ self.__state = self.__waiting
+ elif ttype == tokenize.STRING:
+ self.__data.append(normalize(tstring))
+ # TBD: should we warn if we seen anything else?
+
+ def set_filename(self, filename):
+ self.__curfile = filename
+
+ def write(self, fp):
+ options = self.__options
+ timestamp = time.ctime(time.time())
+ # common header
+ try:
+ sys.stdout = fp
+ print '# POT file generated by pygettext.py', __version__
+ print '#', timestamp
+ print '#'
+ for k, v in self.__messages.items():
+ for filename, lineno in v:
+ # location comments are different b/w Solaris and GNU
+ if options.location == options.SOLARIS:
+ print '# File: %s,' % filename, 'line: %d' % lineno
+ elif options.location == options.GNU:
+ print '#: %s:%d' % (filename, lineno)
+ # TBD: sorting, normalizing
+ print 'msgid', k
+ print 'msgstr '
+ print
+ finally:
+ sys.stdout = sys.__stdout__
+
+
+def main():
+ default_keywords = ['_']
+ try:
+ opts, args = getopt.getopt(
+ sys.argv[1:],
+ 'k:d:n:h',
+ ['keyword', 'default-domain', 'help',
+ 'add-location=', 'no-location'])
+ except getopt.error, msg:
+ usage(1, msg)
+
+ # for holding option values
+ class Options:
+ # constants
+ GNU = 1
+ SOLARIS = 2
+ # defaults
+ keywords = []
+ outfile = 'messages.pot'
+ location = GNU
+
+ options = Options()
+ locations = {'gnu' : options.GNU,
+ 'solaris' : options.SOLARIS,
+ }
+
+ # parse options
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage(0)
+ elif opt in ('-k', '--keyword'):
+ if arg is None:
+ default_keywords = []
+ options.keywords.append(arg)
+ elif opt in ('-d', '--default-domain'):
+ options.outfile = arg + '.pot'
+ elif opt in ('-n', '--add-location'):
+ if arg is None:
+ arg = 'gnu'
+ try:
+ options.location = locations[string.lower(arg)]
+ except KeyError:
+ usage(1, 'Invalid value for --add-location: ' + arg)
+ elif opt in ('--no-location',):
+ options.location = 0
+
+ # calculate all keywords
+ options.keywords.extend(default_keywords)
+
+ # slurp through all the files
+ eater = TokenEater(options)
+ for filename in args:
+ fp = open(filename)
+ eater.set_filename(filename)
+ tokenize.tokenize(fp.readline, eater)
+ fp.close()
+
+ fp = open(options.outfile, 'w')
+ eater.write(fp)
+ fp.close()
+
+
+
+if __name__ == '__main__':
+ main()