Benjamin Peterson | 90f5ba5 | 2010-03-11 22:53:45 +0000 | [diff] [blame] | 1 | #! /usr/bin/env python3 |
Alexander Belopolsky | e8f5832 | 2010-10-15 16:28:20 +0000 | [diff] [blame] | 2 | # Written by Martin v. Löwis <loewis@informatik.hu-berlin.de> |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 3 | |
| 4 | """Generate binary message catalog from textual translation description. |
| 5 | |
| 6 | This program converts a textual Uniforum-style message catalog (.po file) into |
| 7 | a binary GNU catalog (.mo file). This is essentially the same function as the |
| 8 | GNU msgfmt program, however, it is a simpler implementation. |
| 9 | |
| 10 | Usage: msgfmt.py [OPTIONS] filename.po |
| 11 | |
| 12 | Options: |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 13 | -o file |
| 14 | --output-file=file |
| 15 | Specify the output file to write to. If omitted, output will go to a |
| 16 | file named filename.mo (based off the input file name). |
| 17 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 18 | -h |
| 19 | --help |
| 20 | Print this message and exit. |
| 21 | |
| 22 | -V |
| 23 | --version |
| 24 | Display version information and exit. |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 25 | """ |
| 26 | |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 27 | import os |
Ezio Melotti | 9bf379e | 2012-11-09 11:46:19 +0100 | [diff] [blame] | 28 | import sys |
| 29 | import ast |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 30 | import getopt |
| 31 | import struct |
| 32 | import array |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 33 | from email.parser import HeaderParser |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 34 | |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 35 | __version__ = "1.1" |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 36 | |
| 37 | MESSAGES = {} |
| 38 | |
| 39 | |
| 40 | |
| 41 | def usage(code, msg=''): |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 42 | print(__doc__, file=sys.stderr) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 43 | if msg: |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 44 | print(msg, file=sys.stderr) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 45 | sys.exit(code) |
| 46 | |
| 47 | |
| 48 | |
| 49 | def add(id, str, fuzzy): |
| 50 | "Add a non-fuzzy translation to the dictionary." |
| 51 | global MESSAGES |
| 52 | if not fuzzy and str: |
| 53 | MESSAGES[id] = str |
| 54 | |
| 55 | |
| 56 | |
| 57 | def generate(): |
| 58 | "Return the generated output." |
| 59 | global MESSAGES |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 60 | # the keys are sorted in the .mo file |
Georg Brandl | bf82e37 | 2008-05-16 17:02:34 +0000 | [diff] [blame] | 61 | keys = sorted(MESSAGES.keys()) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 62 | offsets = [] |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 63 | ids = strs = b'' |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 64 | for id in keys: |
| 65 | # For each string, we need size and file offset. Each string is NUL |
| 66 | # terminated; the NUL does not count into the size. |
| 67 | offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 68 | ids += id + b'\0' |
| 69 | strs += MESSAGES[id] + b'\0' |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 70 | output = '' |
| 71 | # The header is 7 32-bit unsigned integers. We don't use hash tables, so |
| 72 | # the keys start right after the index tables. |
| 73 | # translated string. |
| 74 | keystart = 7*4+16*len(keys) |
| 75 | # and the values start after the keys |
| 76 | valuestart = keystart + len(ids) |
| 77 | koffsets = [] |
| 78 | voffsets = [] |
| 79 | # The string table first has the list of keys, then the list of values. |
| 80 | # Each entry has first the size of the string, then the file offset. |
| 81 | for o1, l1, o2, l2 in offsets: |
| 82 | koffsets += [l1, o1+keystart] |
| 83 | voffsets += [l2, o2+valuestart] |
| 84 | offsets = koffsets + voffsets |
Martin v. Löwis | 8f0bd56 | 2003-05-09 08:59:17 +0000 | [diff] [blame] | 85 | output = struct.pack("Iiiiiii", |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 86 | 0x950412de, # Magic |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 87 | 0, # Version |
| 88 | len(keys), # # of entries |
| 89 | 7*4, # start of key index |
| 90 | 7*4+len(keys)*8, # start of value index |
| 91 | 0, 0) # size and offset of hash table |
Xtreak | a692efe | 2018-07-21 11:52:12 +0530 | [diff] [blame] | 92 | output += array.array("i", offsets).tobytes() |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 93 | output += ids |
| 94 | output += strs |
| 95 | return output |
| 96 | |
| 97 | |
| 98 | |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 99 | def make(filename, outfile): |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 100 | ID = 1 |
| 101 | STR = 2 |
| 102 | |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 103 | # Compute .mo name from .po name and arguments |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 104 | if filename.endswith('.po'): |
| 105 | infile = filename |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 106 | else: |
| 107 | infile = filename + '.po' |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 108 | if outfile is None: |
| 109 | outfile = os.path.splitext(infile)[0] + '.mo' |
| 110 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 111 | try: |
Xtreak | a692efe | 2018-07-21 11:52:12 +0530 | [diff] [blame] | 112 | with open(infile, 'rb') as f: |
| 113 | lines = f.readlines() |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 114 | except IOError as msg: |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 115 | print(msg, file=sys.stderr) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 116 | sys.exit(1) |
Tim Peters | 182b5ac | 2004-07-18 06:16:08 +0000 | [diff] [blame] | 117 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 118 | section = None |
| 119 | fuzzy = 0 |
| 120 | |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 121 | # Start off assuming Latin-1, so everything decodes without failure, |
| 122 | # until we know the exact encoding |
| 123 | encoding = 'latin-1' |
| 124 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 125 | # Parse the catalog |
| 126 | lno = 0 |
| 127 | for l in lines: |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 128 | l = l.decode(encoding) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 129 | lno += 1 |
| 130 | # If we get a comment line after a msgstr, this is a new entry |
| 131 | if l[0] == '#' and section == STR: |
| 132 | add(msgid, msgstr, fuzzy) |
| 133 | section = None |
| 134 | fuzzy = 0 |
| 135 | # Record a fuzzy mark |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 136 | if l[:2] == '#,' and 'fuzzy' in l: |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 137 | fuzzy = 1 |
| 138 | # Skip comments |
| 139 | if l[0] == '#': |
| 140 | continue |
| 141 | # Now we are in a msgid section, output previous section |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 142 | if l.startswith('msgid') and not l.startswith('msgid_plural'): |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 143 | if section == STR: |
| 144 | add(msgid, msgstr, fuzzy) |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 145 | if not msgid: |
| 146 | # See whether there is an encoding declaration |
| 147 | p = HeaderParser() |
| 148 | charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() |
| 149 | if charset: |
| 150 | encoding = charset |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 151 | section = ID |
| 152 | l = l[5:] |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 153 | msgid = msgstr = b'' |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 154 | is_plural = False |
| 155 | # This is a message with plural forms |
| 156 | elif l.startswith('msgid_plural'): |
| 157 | if section != ID: |
Ezio Melotti | 7c4a7e6 | 2013-08-26 01:32:56 +0300 | [diff] [blame] | 158 | print('msgid_plural not preceded by msgid on %s:%d' % (infile, lno), |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 159 | file=sys.stderr) |
| 160 | sys.exit(1) |
| 161 | l = l[12:] |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 162 | msgid += b'\0' # separator of singular and plural |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 163 | is_plural = True |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 164 | # Now we are in a msgstr section |
| 165 | elif l.startswith('msgstr'): |
| 166 | section = STR |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 167 | if l.startswith('msgstr['): |
| 168 | if not is_plural: |
Martin v. Löwis | 25fcd39 | 2010-07-11 17:39:46 +0000 | [diff] [blame] | 169 | print('plural without msgid_plural on %s:%d' % (infile, lno), |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 170 | file=sys.stderr) |
| 171 | sys.exit(1) |
| 172 | l = l.split(']', 1)[1] |
| 173 | if msgstr: |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 174 | msgstr += b'\0' # Separator of the various plural forms |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 175 | else: |
| 176 | if is_plural: |
Martin v. Löwis | 25fcd39 | 2010-07-11 17:39:46 +0000 | [diff] [blame] | 177 | print('indexed msgstr required for plural on %s:%d' % (infile, lno), |
Martin v. Löwis | cb081b8 | 2010-06-04 18:14:42 +0000 | [diff] [blame] | 178 | file=sys.stderr) |
| 179 | sys.exit(1) |
| 180 | l = l[6:] |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 181 | # Skip empty lines |
| 182 | l = l.strip() |
| 183 | if not l: |
| 184 | continue |
Ezio Melotti | 9bf379e | 2012-11-09 11:46:19 +0100 | [diff] [blame] | 185 | l = ast.literal_eval(l) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 186 | if section == ID: |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 187 | msgid += l.encode(encoding) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 188 | elif section == STR: |
Martin v. Löwis | b6b8110 | 2010-06-04 18:40:55 +0000 | [diff] [blame] | 189 | msgstr += l.encode(encoding) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 190 | else: |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 191 | print('Syntax error on %s:%d' % (infile, lno), \ |
| 192 | 'before:', file=sys.stderr) |
| 193 | print(l, file=sys.stderr) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 194 | sys.exit(1) |
| 195 | # Add last entry |
| 196 | if section == STR: |
| 197 | add(msgid, msgstr, fuzzy) |
| 198 | |
| 199 | # Compute output |
| 200 | output = generate() |
| 201 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 202 | try: |
Xtreak | a692efe | 2018-07-21 11:52:12 +0530 | [diff] [blame] | 203 | with open(outfile,"wb") as f: |
| 204 | f.write(output) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 205 | except IOError as msg: |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 206 | print(msg, file=sys.stderr) |
Tim Peters | 182b5ac | 2004-07-18 06:16:08 +0000 | [diff] [blame] | 207 | |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 208 | |
| 209 | |
| 210 | def main(): |
| 211 | try: |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 212 | opts, args = getopt.getopt(sys.argv[1:], 'hVo:', |
| 213 | ['help', 'version', 'output-file=']) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 214 | except getopt.error as msg: |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 215 | usage(1, msg) |
| 216 | |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 217 | outfile = None |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 218 | # parse options |
| 219 | for opt, arg in opts: |
| 220 | if opt in ('-h', '--help'): |
| 221 | usage(0) |
| 222 | elif opt in ('-V', '--version'): |
Serhiy Storchaka | c56894d | 2013-09-05 17:44:53 +0300 | [diff] [blame] | 223 | print("msgfmt.py", __version__) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 224 | sys.exit(0) |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 225 | elif opt in ('-o', '--output-file'): |
| 226 | outfile = arg |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 227 | # do it |
| 228 | if not args: |
Collin Winter | 6afaeb7 | 2007-08-03 17:06:41 +0000 | [diff] [blame] | 229 | print('No input file given', file=sys.stderr) |
| 230 | print("Try `msgfmt --help' for more information.", file=sys.stderr) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 231 | return |
| 232 | |
| 233 | for filename in args: |
Barry Warsaw | 78d7dc4 | 2001-03-02 16:53:54 +0000 | [diff] [blame] | 234 | make(filename, outfile) |
Barry Warsaw | 72dacb8 | 2000-09-01 08:10:08 +0000 | [diff] [blame] | 235 | |
| 236 | |
| 237 | if __name__ == '__main__': |
| 238 | main() |