forgot to check in the new makeunicodedata.py script
diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 15841d7..3a362ec 100644
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -2,14 +2,16 @@
# (re)generate unicode property and type databases
#
# this script converts a unicode 3.0 database file to
-# Modules/unicodedata_db.h and Objects/unicodetype_db.h
+# Modules/unicodedata_db.h, Modules/unicodename_db.h,
+# and Objects/unicodetype_db.h
#
# history:
# 2000-09-24 fl created (based on bits and pieces from unidb)
# 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table
# 2000-09-25 fl added character type table
-# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields
+# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl expand first/last ranges
+# 2001-01-19 fl added character name tables (2.1)
#
# written by Fredrik Lundh (fredrik@pythonware.com), September 2000
#
@@ -17,7 +19,7 @@
import sys
SCRIPT = sys.argv[0]
-VERSION = "1.1"
+VERSION = "2.1"
UNICODE_DATA = "UnicodeData-Latest.txt"
@@ -42,18 +44,32 @@
def maketables(trace=0):
+ print "--- Reading", UNICODE_DATA, "..."
+
unicode = UnicodeData(UNICODE_DATA)
- print "--- Processing", UNICODE_DATA, "..."
print len(filter(None, unicode.table)), "characters"
- # extract unicode properties
+ makeunicodedata(unicode, trace)
+ makeunicodetype(unicode, trace)
+ makeunicodename(unicode, trace)
+
+# --------------------------------------------------------------------
+# unicode character properties
+
+def makeunicodedata(unicode, trace):
+
dummy = (0, 0, 0, 0)
table = [dummy]
cache = {0: dummy}
index = [0] * len(unicode.chars)
+ FILE = "Modules/unicodedata_db.h"
+
+ print "--- Preparing", FILE, "..."
+
# 1) database properties
+
for char in unicode.chars:
record = unicode.table[char]
if record:
@@ -93,13 +109,11 @@
i = 0
decomp_index[char] = i
- FILE = "Modules/unicodedata_db.h"
-
- print "--- Writing", FILE, "..."
-
print len(table), "unique properties"
print len(decomp_data), "unique decomposition entries"
+ print "--- Writing", FILE, "..."
+
fp = open(FILE, "w")
print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
print >>fp
@@ -111,7 +125,7 @@
print >>fp, "};"
print >>fp
- # FIXME: the following tables should be made static, and
+ # FIXME: <fl> the following tables could be made static, and
# the support code moved into unicodedatabase.c
print >>fp, "/* string literals */"
@@ -149,8 +163,16 @@
Array("decomp_index1", index1).dump(fp)
Array("decomp_index2", index2).dump(fp)
- #
- # 3) unicode type data
+ fp.close()
+
+# --------------------------------------------------------------------
+# unicode character type tables
+
+def makeunicodetype(unicode, trace):
+
+ FILE = "Objects/unicodetype_db.h"
+
+ print "--- Preparing", FILE, "..."
# extract unicode types
dummy = (0, 0, 0, 0, 0, 0)
@@ -209,14 +231,11 @@
table.append(item)
index[char] = i
- FILE = "Objects/unicodetype_db.h"
-
- fp = open(FILE, "w")
+ print len(table), "unique character type entries"
print "--- Writing", FILE, "..."
- print len(table), "unique character type entries"
-
+ fp = open(FILE, "w")
print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
print >>fp
print >>fp, "/* a list of unique character type descriptors */"
@@ -234,6 +253,155 @@
Array("index1", index1).dump(fp)
Array("index2", index2).dump(fp)
+ fp.close()
+
+# --------------------------------------------------------------------
+# unicode name database
+
+def makeunicodename(unicode, trace):
+
+ FILE = "Modules/unicodename_db.h"
+
+ print "--- Preparing", FILE, "..."
+
+ # collect names
+ names = [None] * len(unicode.chars)
+
+ for char in unicode.chars:
+ record = unicode.table[char]
+ if record:
+ name = record[1].strip()
+ if name and name[0] != "<":
+ names[char] = name + chr(0)
+
+ print len(filter(lambda n: n is not None, names)), "distinct names"
+
+ # collect unique words from names (note that we differ between
+ # words inside a sentence, and words ending a sentence. the
+ # latter includes the trailing null byte.
+
+ words = {}
+ n = b = 0
+ for char in unicode.chars:
+ name = names[char]
+ if name:
+ w = name.split()
+ b = b + len(name)
+ n = n + len(w)
+ for w in w:
+ l = words.get(w)
+ if l:
+ l.append(None)
+ else:
+ words[w] = [len(words)]
+
+ print n, "words in text;", b, "bytes"
+
+ wordlist = words.items()
+
+ # sort on falling frequency
+ wordlist.sort(lambda a, b: len(b[1])-len(a[1]))
+
+ # statistics
+ n = 0
+ for i in range(128):
+ n = n + len(wordlist[i][1])
+ print n, "short words (7-bit indices)"
+
+ # pick the 128 most commonly used words, and sort the rest on
+ # falling length (to maximize overlap)
+
+ wordlist, wordtail = wordlist[:128], wordlist[128:]
+ wordtail.sort(lambda a, b: len(b[0])-len(a[0]))
+ wordlist.extend(wordtail)
+
+ # generate lexicon from words
+
+ lexicon_offset = [0]
+ lexicon = ""
+ words = {}
+
+ # build a lexicon string
+ offset = 0
+ for w, x in wordlist:
+ # encoding: bit 7 indicates last character in word (chr(128)
+ # indicates the last character in an entire string)
+ ww = w[:-1] + chr(ord(w[-1])+128)
+ # reuse string tails, when possible
+ o = string.find(lexicon, ww)
+ if o < 0:
+ o = offset
+ lexicon = lexicon + ww
+ offset = offset + len(w)
+ words[w] = len(lexicon_offset)
+ lexicon_offset.append(offset)
+
+ print len(words), "words in lexicon;", len(lexicon), "bytes"
+
+ assert len(words) < 32768 # 15-bit word indices
+
+ lexicon = map(ord, lexicon)
+
+ # generate phrasebook from names and lexicon
+ phrasebook = [0]
+ phrasebook_offset = [0] * len(unicode.chars)
+ for char in unicode.chars:
+ name = names[char]
+ if name:
+ w = name.split()
+ phrasebook_offset[char] = len(phrasebook)
+ for w in w:
+ i = words[w]
+ if i < 128:
+ phrasebook.append(128+i)
+ else:
+ phrasebook.append(i>>8)
+ phrasebook.append(i&255)
+
+ #
+ # unicode name hash table
+
+ # extract names
+ data = []
+ for char in unicode.chars:
+ record = unicode.table[char]
+ if record:
+ name = record[1].strip()
+ if name and name[0] != "<":
+ data.append((name, char))
+
+ # the magic number 47 was chosen to minimize the number of
+ # collisions on the current data set. if you like, change it
+ # and see what happens...
+
+ codehash = Hash("code", data, 47)
+
+ print "--- Writing", FILE, "..."
+
+ fp = open(FILE, "w")
+ print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
+ print >>fp
+ print >>fp, "#define NAME_MAXLEN", 256
+ print >>fp
+ print >>fp, "/* lexicon */"
+ Array("lexicon", lexicon).dump(fp)
+ Array("lexicon_offset", lexicon_offset).dump(fp)
+
+ # split decomposition index table
+ offset1, offset2, shift = splitbins(phrasebook_offset, trace)
+
+ print >>fp, "/* code->name phrasebook */"
+ print >>fp, "#define phrasebook_shift", shift
+
+ Array("phrasebook", phrasebook).dump(fp)
+ Array("phrasebook_offset1", offset1).dump(fp)
+ Array("phrasebook_offset2", offset2).dump(fp)
+
+ print >>fp, "/* name->code dictionary */"
+ codehash.dump(fp)
+
+ fp.close()
+
# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB
@@ -280,6 +448,92 @@
# restrict character range to ISO Latin 1
self.chars = range(256)
+# hash table tools
+
+# this is a straight-forward reimplementation of Python's built-in
+# dictionary type, using a static data structure, and a custom string
+# hash algorithm.
+
+def myhash(s, magic):
+ h = 0
+ for c in map(ord, string.upper(s)):
+ h = (h * magic) + c
+ ix = h & 0xff000000
+ if ix:
+ h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff
+ return h
+
+SIZES = [
+ (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17),
+ (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3),
+ (65536,45), (131072,9), (262144,39), (524288,39), (1048576,9),
+ (2097152,5), (4194304,3), (8388608,33), (16777216,27)
+]
+
+class Hash:
+ def __init__(self, name, data, magic):
+ # turn a (key, value) list into a static hash table structure
+
+ # determine table size
+ for size, poly in SIZES:
+ if size > len(data):
+ poly = size + poly
+ break
+ else:
+ raise AssertionError, "ran out of polynominals"
+
+ print size, "slots in hash table"
+
+ table = [None] * size
+
+ mask = size-1
+
+ n = 0
+
+ hash = myhash
+
+ # initialize hash table
+ for key, value in data:
+ h = hash(key, magic)
+ i = (~h) & mask
+ v = table[i]
+ if v is None:
+ table[i] = value
+ continue
+ incr = (h ^ (h >> 3)) & mask;
+ if not incr:
+ incr = mask
+ while 1:
+ n = n + 1
+ i = (i + incr) & mask
+ v = table[i]
+ if v is None:
+ table[i] = value
+ break
+ incr = incr << 1
+ if incr > mask:
+ incr = incr ^ poly
+
+ print n, "collisions"
+ self.collisions = n
+
+ for i in range(len(table)):
+ if table[i] is None:
+ table[i] = 0
+
+ self.data = Array(name + "_hash", table)
+ self.magic = magic
+ self.name = name
+ self.size = size
+ self.poly = poly
+
+ def dump(self, file):
+ # write data to file, as a C array
+ self.data.dump(file)
+ file.write("#define %s_magic %d\n" % (self.name, self.magic))
+ file.write("#define %s_size %d\n" % (self.name, self.size))
+ file.write("#define %s_poly %d\n" % (self.name, self.poly))
+
# stuff to deal with arrays of unsigned integers
class Array: