| /* |
| * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. Oracle designates this |
| * particular file as subject to the "Classpath" exception as provided |
| * by Oracle in the LICENSE file that accompanied this code. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| package build.tools.generatecharacter; |
| |
| import java.io.IOException; |
| import java.io.FileNotFoundException; |
| import java.io.BufferedReader; |
| import java.io.FileReader; |
| import java.io.PrintWriter; |
| import java.io.BufferedWriter; |
| import java.io.FileWriter; |
| import java.io.File; |
| import java.util.List; |
| |
| import build.tools.generatecharacter.CharacterName; |
| |
| /** |
| * This program generates the source code for the class java.lang.Character. |
| * It also generates native C code that can perform the same operations. |
| * It requires two external input data files: |
| * <ul> |
| * <li> Unicode specification file |
| * <li> Character class template file |
| * </ul> |
| * The Unicode specification file is available from the Unicode consortium. |
| * It has character specification lines that look like this: |
| * <listing> |
| * 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061; |
| * </listing> |
| * The Character class template file is filled in with additional |
| * information to produce the file Character.java, which can then be |
| * compiled by a Java compiler. The template file contains certain |
| * markers consisting of an alphabetic name string preceded by "$$". |
| * Such markers are replaced with generated program text. As a special |
| * case, the marker "Lookup(xxx)" is recognized, where "xxx" consists of |
| * alphabetic characters constituting a variable name. The character "_" |
| * is considered alphabetic for these purposes. |
| * |
| * @author Guy Steele |
| * @author Alan Liu |
| * @author John O'Conner |
| */ |
| |
| public class GenerateCharacter { |
| |
| final static boolean DEBUG = false; |
| |
| final static String commandMarker = "$$"; |
| static String ROOT = ""; |
| static String DefaultUnicodeSpecFileName = ROOT + "UnicodeData.txt"; |
| static String DefaultSpecialCasingFileName = ROOT + "SpecialCasing.txt"; |
| static String DefaultPropListFileName = ROOT + "PropList.txt"; |
| static String DefaultJavaTemplateFileName = ROOT + "Character.java.template"; |
| static String DefaultJavaOutputFileName = ROOT + "Character.java"; |
| static String DefaultCTemplateFileName = ROOT + "Character.c.template"; |
| static String DefaultCOutputFileName = ROOT + "Character.c"; |
| |
| static int plane = 0; |
| |
| /* The overall idea is that, in the generated Character class source code, |
| most character property data is stored in a special multi-level table whose |
| structure is defined by a sequence of nonnegative integers [k1, k2, ..., kn]. |
| The integers must sum to 16 (the number of bits in a character). |
| The first table is indexed by the k1 high-order bits of the character code. |
| The result is concatenated to the next k2 bits of the character code to index |
| the second table, and so on. Eventually the kn low-order bits of the character |
| code are concatenated and used to index one of two tables A and B; A contains |
| 32-bit integer entries and B contains 16-bit short entries. The 48 bits that |
| can be thus obtained encode the properties for the character. |
| |
| The default specification is [9, 4, 3, 0]. This particular table format was |
| designed by conducting an exhaustive search of table formats to minimize the |
| space consumed by the tables: the first and third tables need have only byte |
| values (the second table must have short values). Another good choice is |
| [10, 6, 0], which produces a larger table but allows particularly fast table |
| lookup code. |
| |
| In each case, where the word "concatenated" is used, this may imply |
| first a << and then a | operation, or perhaps just a | operation if |
| the values in the table can be preshifted (generally possible if the table |
| entries are short rather than byte). |
| */ |
| |
| /* The character properties are currently encoded into A (32 bits)and B (16 bits) |
| two parts. |
| |
| A: the low 32 bits are defined in the following manner: |
| |
| 1 bit Mirrored property. |
| 4 bits Bidirectional category (see below) (unused if -nobidi switch specified) |
| 9 bits A signed offset used for converting case . |
| 1 bit If 1, adding the signed offset converts the character to lowercase. |
| 1 bit If 1, subtracting the signed offset converts the character to uppercase. |
| Note: for a titlecase character, both of the preceding bits will be 1 |
| and the signed offset will be 1. |
| 1 bit If 1, this character has a titlecase equivalent (possibly itself); |
| in this case, the two bits before this bit can be used to decide |
| whether this character is in fact uppercase, lowercase, or titlecase. |
| 3 bits This field provides a quick way to lex identifiers. |
| The eight possible values for this field are as follows: |
| 0 May not be part of an identifier |
| 1 Ignorable control; may continue a Unicode identifier or Java identifier |
| 2 May continue a Java identifier but not a Unicode identifier (unused) |
| 3 May continue a Unicode identifier or Java identifier |
| 4 Is a Java whitespace character |
| 5 May start or continue a Java identifier; |
| may continue but not start a Unicode identifier |
| (this value is used for connector punctuation such as _) |
| 6 May start or continue a Java identifier; |
| may not occur in a Unicode identifier |
| (this value is used for currency symbols such as $) |
| 7 May start or continue a Unicode identifier or Java identifier |
| Thus: |
| 5, 6, 7 may start a Java identifier |
| 1, 2, 3, 5, 6, 7 may continue a Java identifier |
| 7 may start a Unicode identifier |
| 1, 3, 5, 7 may continue a Unicode identifier |
| 1 is ignorable within an identifier |
| 4 is Java whitespace |
| 2 bits This field indicates whether the character has a numeric property. |
| The four possible values for this field are as follows: |
| 0 This character has no numeric property. |
| 1 Adding the digit offset to the character code and then |
| masking with 0x1F will produce the desired numeric value. |
| 2 This character has a "strange" numeric value. |
| 3 A Java supradecimal digit: adding the digit offset to the |
| character code, then masking with 0x1F, then adding 10 |
| will produce the desired numeric value. |
| 5 bits The digit offset (see description of previous field) |
| 5 bits Character type (see below) |
| |
| B: the high 16 bits are defined as: |
| 1 bit Other_Lowercase property |
| 1 bit Other_Uppercase property |
| 1 bit Other_Alphabetic property |
| 1 bit Other_Math property |
| 1 bit Ideographic property |
| 1 bit Noncharacter codepoint property |
| */ |
| |
| |
| // bit masks identify each component of a 32-bit property field described |
| // above. |
| // shift* indicates how many shifts right must happen to get the |
| // indicated property value in the lowest bits of the 32-bit space. |
| private static final int |
| shiftType = 0, maskType = 0x001F, |
| shiftDigitOffset = 5, maskDigitOffset = 0x03E0, |
| shiftNumericType = 10, maskNumericType = 0x0C00, |
| shiftIdentifierInfo = 12, maskIdentifierInfo = 0x7000, |
| maskUnicodePart = 0x1000, |
| shiftCaseInfo = 15, maskCaseInfo = 0x38000, |
| maskLowerCase = 0x20000, |
| maskUpperCase = 0x10000, |
| maskTitleCase = 0x08000, |
| shiftCaseOffset = 18, maskCaseOffset = 0x07FC0000, |
| shiftCaseOffsetSign = 5, |
| // used only when calculating and |
| // storing digit offsets from char values |
| maskDigit = 0x001F, |
| // case offset are 9 bits |
| maskCase = 0x01FF, |
| shiftBidi = 27, maskBidi = 0x78000000, |
| shiftMirrored = 31, //maskMirrored = 0x80000000, |
| shiftPlane = 16, maskPlane = 0xFF0000; |
| |
| // maskMirrored needs to be long, if up 16-bit |
| private static final long maskMirrored = 0x80000000L; |
| |
| // bit masks identify the 16-bit priperty field described above, in B |
| // table |
| private static final long |
| maskOtherLowercase = 0x100000000L, |
| maskOtherUppercase = 0x200000000L, |
| maskOtherAlphabetic = 0x400000000L, |
| maskOtherMath = 0x800000000L, |
| maskIdeographic = 0x1000000000L, |
| maskNoncharacterCP = 0x2000000000L; |
| |
| // Can compare masked values with these to determine |
| // numeric or lexical types. |
| public static int |
| valueNotNumeric = 0x0000, |
| valueDigit = 0x0400, |
| valueStrangeNumeric = 0x0800, |
| valueJavaSupradecimal = 0x0C00, |
| valueIgnorable = 0x1000, |
| valueJavaOnlyPart = 0x2000, |
| valueJavaUnicodePart = 0x3000, |
| valueJavaWhitespace = 0x4000, |
| valueJavaStartUnicodePart = 0x5000, |
| valueJavaOnlyStart = 0x6000, |
| valueJavaUnicodeStart = 0x7000, |
| lowJavaStart = 0x5000, |
| nonzeroJavaPart = 0x3000, |
| valueUnicodeStart = 0x7000; |
| |
| // these values are used when only identifier properties are generated |
| // for use in verifier code. Shortens the property down to a single byte. |
| private static final int |
| bitJavaStart = 0x02, |
| bitJavaPart = 0x01, |
| maskIsJavaIdentifierPart = bitJavaPart, |
| maskIsJavaIdentifierStart = bitJavaStart; |
| |
| static int maxOffset = maskCase/2 ; |
| static int minOffset = -maxOffset; |
| |
| /* The following routines provide simple, concise formatting of long integer values. |
| The number in the name of the method indicates the desired number of characters |
| to be produced. If the number of digits required to represent the integer value |
| is less than that number, then the output is padded on the left with zeros |
| (for hex) or with spaces (for decimal). If the number of digits required to |
| represent the integer value is greater than the desired number, then all the digits |
| that are required are actually produced. |
| */ |
| |
| static String hex(long n) { return Long.toHexString(n).toUpperCase(); } |
| |
| static String hex2(long n) { |
| String q = Long.toHexString(n & 0xFF).toUpperCase(); |
| return "00".substring(Math.min(2, q.length())) + q; |
| } |
| |
| static String hex4(long n) { |
| String q = Long.toHexString(n & 0xFFFF).toUpperCase(); |
| return "0000".substring(Math.min(4, q.length())) + q; |
| } |
| |
| static String hex8(long n) { |
| String q = Long.toHexString(n & 0xFFFFFFFFL).toUpperCase(); |
| return "00000000".substring(Math.min(8, q.length())) + q; |
| } |
| |
| static String hex16(long n) { |
| String q = Long.toHexString(n).toUpperCase(); |
| return "0000000000000000".substring(Math.min(16, q.length())) + q; |
| } |
| |
| static String dec3(long n) { |
| String q = Long.toString(n); |
| return " ".substring(Math.min(3, q.length())) + q; |
| } |
| |
| static String dec5(long n) { |
| String q = Long.toString(n); |
| return " ".substring(Math.min(5, q.length())) + q; |
| } |
| |
| /* This routine is called when some failure occurs. */ |
| |
| static void FAIL(String s) { |
| System.out.println("** " + s); |
| } |
| |
| /** |
| * Given the data from the Unicode specification file, this routine builds a map. |
| * |
| * The specification file is assumed to contain its data in sorted order by |
| * character code; as a result, the array passed as an argument to this method |
| * has its components in the same sorted order, with one entry for each defined |
| * Unicode character or character range. (A range is indicated by two consecutive |
| * entries, such that the name of the first entry begins with "<" and ends with |
| * "First>" and the second entry begins with "<" and ends with "Last>".) This is |
| * therefore a sparse representation of the character property data. |
| * |
| * The resulting map is dense representation of the character data. It contains |
| * 2^16 = 65536 entries, each of which is a long integer. (Right now only 32 bits |
| * of this long value are used, but type long is used rather than int to facilitate |
| * future extensions of this source code generator that might require more than |
| * 32 bits to encode relevant character properties.) Entry k holds the encoded |
| * properties for character k. |
| * |
| * Method buildMap manages the transformation from the sparse representation to |
| * the dense representation. It calls method buildOne to handle the encoding |
| * of character property data from a single UnicodeSpec object into 32 bits. |
| * For undefined characters, method buildOne is not called and the map entry for |
| * that character is set to UnicodeSpec.UNASSIGNED. |
| * |
| * @param data character property data from the Unicode specification file |
| * @return an array of length 65536 with one entry for every possible char value |
| * |
| * @see GenerateCharacter#buildOne |
| */ |
| |
| static long[] buildMap(UnicodeSpec[] data, SpecialCaseMap[] specialMaps, PropList propList) |
| { |
| long[] result; |
| if (bLatin1 == true) { |
| result = new long[256]; |
| } else { |
| result = new long[1<<16]; |
| } |
| int k=0; |
| int codePoint = plane<<16; |
| UnicodeSpec nonCharSpec = new UnicodeSpec(); |
| for (int j = 0; j < data.length && k < result.length; j++) { |
| if (data[j].codePoint == codePoint) { |
| result[k] = buildOne(codePoint, data[j], specialMaps); |
| ++k; |
| ++codePoint; |
| } |
| else if(data[j].codePoint > codePoint) { |
| if (data[j].name.endsWith("Last>")) { |
| // build map data for all chars except last in range |
| while (codePoint < data[j].codePoint && k < result.length) { |
| result[k] = buildOne(codePoint, data[j], specialMaps); |
| ++k; |
| ++codePoint; |
| } |
| } |
| else { |
| // we have a few unassigned chars before data[j].codePoint |
| while (codePoint < data[j].codePoint && k < result.length) { |
| result[k] = buildOne(codePoint, nonCharSpec, specialMaps); |
| ++k; |
| ++codePoint; |
| } |
| } |
| k = data[j].codePoint & 0xFFFF; |
| codePoint = data[j].codePoint; |
| result[k] = buildOne(codePoint, data[j], specialMaps); |
| ++k; |
| ++codePoint; |
| } |
| else { |
| System.out.println("An error has occured during spec mapping."); |
| System.exit(0); |
| } |
| } |
| // if there are still unprocessed chars, process them |
| // as unassigned/undefined. |
| codePoint = (plane<<16) | k; |
| while (k < result.length) { |
| result[k] = buildOne(codePoint, nonCharSpec, specialMaps); |
| ++k; |
| ++codePoint; |
| } |
| // now add all extra supported properties from PropList, to the |
| // upper 16-bit |
| addExProp(result, propList, "Other_Lowercase", maskOtherLowercase); |
| addExProp(result, propList, "Other_Uppercase", maskOtherUppercase); |
| addExProp(result, propList, "Other_Alphabetic", maskOtherAlphabetic); |
| addExProp(result, propList, "Ideographic", maskIdeographic); |
| //addExProp(result, propList, "Other_Math", maskOtherMath); |
| //addExProp(result, propList, "Noncharacter_CodePoint", maskNoncharacterCP); |
| |
| return result; |
| } |
| |
| // The maximum and minimum offsets found while scanning the database |
| static int maxOffsetSeen = 0; |
| static int minOffsetSeen = 0; |
| |
| /** |
| * Some Unicode separator characters are not considered Java whitespace. |
| * @param c character to test |
| * @return true if c in an invalid Java whitespace character, false otherwise. |
| */ |
| static boolean isInvalidJavaWhiteSpace(int c) { |
| int[] exceptions = {0x00A0, 0x2007, 0x202F, 0xFEFF}; |
| boolean retValue = false; |
| for(int x=0;x<exceptions.length;x++) { |
| if(c == exceptions[x]) { |
| retValue = true; |
| break; |
| } |
| } |
| return retValue; |
| |
| } |
| |
| /** |
| * Given the character property data for one Unicode character, encode the data |
| * of interest into a single long integer value. (Right now only 32 bits |
| * of this long value are used, but type long is used rather than int to facilitate |
| * future extensions of this source code generator that might require more than |
| * 32 bits to encode relevant character properties.) |
| * |
| * @param c the character code for which to encode property data |
| * @param us property data record from the Unicode specification file |
| * (its character code might not be equal to c if it specifies data |
| * for a range of characters) |
| * @return an encoded long value that contains the properties for a single char |
| * |
| * @see GenerateCharacter#buildMap |
| */ |
| |
| static long buildOne(int c, UnicodeSpec us, SpecialCaseMap[] specialMaps) { |
| long resultA = 0; |
| // record the general category |
| resultA |= us.generalCategory; |
| |
| // record the numeric properties |
| NUMERIC: { |
| STRANGE: { |
| int val = 0; |
| // c is A-Z |
| if ((c >= 0x0041) && (c <= 0x005A)) { |
| val = c - 0x0041; |
| resultA |= valueJavaSupradecimal; |
| // c is a-z |
| } else if ((c >= 0x0061) && (c <= 0x007A)) { |
| val = c - 0x0061; |
| resultA |= valueJavaSupradecimal; |
| // c is a full-width A-Z |
| } else if ((c >= 0xFF21) && (c <= 0xFF3A)) { |
| val = c - 0xFF21; |
| resultA |= valueJavaSupradecimal; |
| // c is a full-width a-z |
| } else if ((c >= 0xFF41) && (c <= 0xFF5A)) { |
| val = c - 0xFF41; |
| resultA |= valueJavaSupradecimal; |
| } else if (us.isDecimalValue()) { |
| val = us.decimalValue; |
| resultA |= valueDigit; |
| } else if (us.isDigitValue()) { |
| val = us.digitValue; |
| resultA |= valueDigit; |
| } else { |
| if (us.numericValue.length() == 0) { |
| break NUMERIC; // no numeric value at all |
| } else { |
| try { |
| val = Integer.parseInt(us.numericValue); |
| if (val >= 32 || val < 0) break STRANGE; |
| if (c == 0x215F) break STRANGE; |
| } catch(NumberFormatException e) { |
| break STRANGE; |
| } |
| resultA |= valueDigit; |
| } |
| } |
| if (val >= 32 || val < 0) break STRANGE; |
| resultA |= ((val - c & maskDigit) << shiftDigitOffset); |
| break NUMERIC; |
| } // end STRANGE |
| resultA |= valueStrangeNumeric; |
| } // end NUMERIC |
| |
| // record case mapping |
| int offset = 0; |
| // might have a 1:M mapping |
| int specialMap = SpecialCaseMap.find(c, specialCaseMaps); |
| boolean bHasUpper = (us.hasUpperMap()) || (specialMap != -1); |
| if (bHasUpper) { |
| resultA |= maskUpperCase; |
| } |
| if (specialMap != -1) { |
| // has mapping, but cannot record the |
| // proper offset; can only flag it and provide special case |
| // code in Character.java |
| offset = -1; |
| } |
| else if (us.hasUpperMap()) { |
| offset = c - us.upperMap; |
| } |
| |
| if (us.hasLowerMap()) { |
| resultA |= maskLowerCase; |
| if (offset == 0) |
| offset = us.lowerMap - c; |
| else if (offset != (us.lowerMap - c)) { |
| if (DEBUG) { |
| FAIL("Character " + hex(c) + |
| " has incompatible lowercase and uppercase mappings"); |
| } |
| } |
| } |
| if ((us.hasTitleMap() && us.titleMap != us.upperMap) || |
| (bHasUpper && us.hasLowerMap())) { |
| resultA |= maskTitleCase; |
| } |
| if (bHasUpper && !us.hasLowerMap() && !us.hasTitleMap() && verbose) { |
| System.out.println("Warning: Character " + hex4(c) + " has upper but " + |
| "no title case; Java won't know this"); |
| } |
| if (offset < minOffsetSeen) minOffsetSeen = offset; |
| if (offset > maxOffsetSeen) maxOffsetSeen = offset; |
| if (offset > maxOffset || offset < minOffset) { |
| if (DEBUG) { |
| FAIL("Case offset " + offset + " for character " + hex4(c) + " must be handled as a special case"); |
| } |
| offset = maskCase; |
| } |
| resultA |= ((offset & maskCase) << shiftCaseOffset); |
| |
| // record lexical info about this character |
| if (us.generalCategory == UnicodeSpec.LOWERCASE_LETTER |
| || us.generalCategory == UnicodeSpec.UPPERCASE_LETTER |
| || us.generalCategory == UnicodeSpec.TITLECASE_LETTER |
| || us.generalCategory == UnicodeSpec.MODIFIER_LETTER |
| || us.generalCategory == UnicodeSpec.OTHER_LETTER |
| || us.generalCategory == UnicodeSpec.LETTER_NUMBER) { |
| resultA |= valueJavaUnicodeStart; |
| } |
| else if (us.generalCategory == UnicodeSpec.COMBINING_SPACING_MARK |
| || us.generalCategory == UnicodeSpec.NON_SPACING_MARK |
| || us.generalCategory == UnicodeSpec.DECIMAL_DIGIT_NUMBER) { |
| resultA |= valueJavaUnicodePart; |
| } |
| else if (us.generalCategory == UnicodeSpec.CONNECTOR_PUNCTUATION) { |
| resultA |= valueJavaStartUnicodePart; |
| } |
| else if (us.generalCategory == UnicodeSpec.CURRENCY_SYMBOL) { |
| resultA |= valueJavaOnlyStart; |
| } |
| else if (((c >= 0x0000) && (c <= 0x0008)) |
| || ((c >= 0x000E) && (c <= 0x001B)) |
| || ((c >= 0x007F) && (c <= 0x009F)) |
| || us.generalCategory == UnicodeSpec.FORMAT) { |
| resultA |= valueIgnorable; |
| } |
| else if (us.generalCategory == UnicodeSpec.SPACE_SEPARATOR |
| || us.generalCategory == UnicodeSpec.LINE_SEPARATOR |
| || us.generalCategory == UnicodeSpec.PARAGRAPH_SEPARATOR) { |
| if (!isInvalidJavaWhiteSpace(c)) resultA |= valueJavaWhitespace; |
| } |
| else if (((c >= 0x0009) && (c <= 0x000D)) |
| || ((c >= 0x001C) && (c <= 0x001F))) { |
| resultA |= valueJavaWhitespace; |
| } |
| |
| // record bidi category |
| if (!nobidi) { |
| int tmpBidi = |
| (us.bidiCategory > UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS || |
| us.bidiCategory == -1) ? maskBidi : (us.bidiCategory << shiftBidi); |
| resultA |= tmpBidi; |
| } |
| |
| // record mirrored property |
| if (!nomirror) { |
| resultA |= us.mirrored ? maskMirrored : 0; |
| } |
| |
| if (identifiers) { |
| long replacement = 0; |
| if ((resultA & maskIdentifierInfo) >= lowJavaStart) { |
| replacement |= bitJavaStart; |
| } |
| if ( ((resultA & nonzeroJavaPart) != 0) |
| && ((resultA & maskIdentifierInfo) != valueIgnorable)) { |
| replacement |= bitJavaPart; |
| } |
| resultA = replacement; |
| } |
| return resultA; |
| } |
| |
| static void addExProp(long[] map, PropList propList, String prop, long mask) { |
| List<Integer> cps = propList.codepoints(prop); |
| if (cps != null) { |
| for (Integer cp : cps) { |
| if (cp < map.length) |
| map[cp] |= mask; |
| } |
| } |
| } |
| |
| /** |
| * This is the heart of the table compression strategy. The inputs are a map |
| * and a number of bits (size). The map is simply an array of long integer values; |
| * the number of bits indicates how index values for that map are to be split. |
| * The length of the given map must be a multiple of (1 << size). The result is |
| * a new map z and a compressed table t such that for every valid index value k |
| * for the original map, t[(z[k>>size]<<size)|(k & ((1<<size)-1))] == map[k]. |
| * |
| * In other words, the index k can be split into two parts, namely the "size" |
| * low-order bits and all the remaining high-order bits; the high-order bits are then |
| * remapped by map z to produce an index into table t. In effect, the data of the |
| * original map m is broken up into blocks of size (1<<size); the compression relies |
| * on the expectation that many of these blocks will be identical and therefore need |
| * be represented only once in the compressed table t. |
| * |
| * This method is intended to be used iteratively. The first map to be handed |
| * to it is the one constructed by method buildMap. After that, the first of the |
| * two arrays returned by this method is fed back into it for further compression. |
| * At the end of the iteration, one has a starter map and a sequence of tables. |
| * |
| * The algorithm used to implement this computation is straightforward and not |
| * especially clever. It uses brute-force linear search (the loop labeled MIDDLE) |
| * to locate identical blocks, so overall the time complexity of the algorithm |
| * is quadratic in the length of the input map. Fortunately, speed is not crucial |
| * to this application. |
| * |
| * @param map a map to be compressed |
| * @param size the number of index bits to be split off by the compression |
| * @return an array of length 2 containing two arrays; the first is a new map |
| * and the second is a compressed data table |
| * |
| * @see GenerateCharacter#buildMap |
| */ |
| |
| static long[][] buildTable(long[] map, int size) { |
| int n = map.length; |
| if (((n >> size) << size) != n) { |
| FAIL("Length " + n + " is not a multiple of " + (1 << size)); |
| } |
| int m = 1 << size; |
| // We know the final length of the new map up front. |
| long[] newmap = new long[n >> size]; |
| // The buffer is used temporarily to hold data for the compressed table |
| // because we don't know its final length yet. |
| long[] buffer = new long[n]; |
| int ptr = 0; |
| OUTER: for (int i = 0; i < n; i += m) { |
| // For every block of size m in the original map... |
| MIDDLE: for (int j = 0; j < ptr; j += m) { |
| // Find out whether there is already a block just like it in the buffer. |
| for (int k = 0; k < m; k++) { |
| if (buffer[j+k] != map[i+k]) |
| continue MIDDLE; |
| } |
| // There is a block just like it at position j, so just |
| // put its index into the new map (thereby sharing it). |
| newmap[i >> size] = (j >> size); |
| continue OUTER; |
| } // end MIDDLE |
| // There is no block just like it already, so add it to |
| // the buffer and put its index into the new map. |
| for (int k = 0; k < m; k++) { |
| buffer[ptr+k] = map[i+k]; |
| } |
| newmap[i >> size] = (ptr >> size); |
| ptr += m; |
| } // end OUTER |
| // Now we know how long the compressed table should be, |
| // so create a new array and copy data from the temporary buffer. |
| long[] newdata = new long[ptr]; |
| for (int j = 0; j < ptr; j++) { |
| newdata[j] = buffer[j]; |
| } |
| // Return the new map and the new data table. |
| long[][] result = { newmap, newdata }; |
| return result; |
| } |
| |
| /** |
| * Once the compressed tables have been computed, this method reads in a |
| * template file for the source code to be generated and writes out the final |
| * source code by acting as a sort of specialized macro processor. |
| * |
| * The first output line is a comment saying that the file was automatically |
| * generated; it includes a timestamp. All other output is generated by |
| * reading a line from the template file, performing macro replacements, |
| * and then writing the resulting line or lines of code to the output file. |
| * |
| * This method handles the I/O, the timestamp comment, and the locating of |
| * macro calls within each input line. The method replaceCommand is called |
| * to generate replacement text for each macro call. |
| * |
| * Macro calls to be replaced are indicated in the template file by |
| * occurrences of the commandMarker "$$". The rest of the call may consist |
| * of Java letters (including the underscore "_") and also of balanced |
| * parentheses. |
| * |
| * @param theTemplateFileName |
| * the file name for the template input file |
| * @param theOutputFileName |
| * the file name for the source code output file |
| * |
| * @see GenerateCharacter#replaceCommand |
| */ |
| |
| static void generateCharacterClass(String theTemplateFileName, |
| String theOutputFileName) |
| throws FileNotFoundException, IOException { |
| BufferedReader in = new BufferedReader(new FileReader(theTemplateFileName)); |
| PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(theOutputFileName))); |
| out.println(commentStart + |
| " This file was generated AUTOMATICALLY from a template file " + |
| new java.util.Date() + commentEnd); |
| int marklen = commandMarker.length(); |
| LOOP: while(true) { |
| try { |
| String line = in.readLine(); |
| if (line == null) break LOOP; |
| int pos = 0; |
| int depth = 0; |
| while ((pos = line.indexOf(commandMarker, pos)) >= 0) { |
| int newpos = pos + marklen; |
| char ch = 'x'; |
| SCAN: while (newpos < line.length() && |
| (Character.isJavaIdentifierStart(ch = line.charAt(newpos)) |
| || ch == '(' || (ch == ')' && depth > 0))) { |
| ++newpos; |
| if (ch == '(') { |
| ++depth; |
| } |
| else if (ch == ')') { |
| --depth; |
| if (depth == 0) |
| break SCAN; |
| } |
| } |
| String replacement = replaceCommand(line.substring(pos + marklen, newpos)); |
| line = line.substring(0, pos) + replacement + line.substring(newpos); |
| pos += replacement.length(); |
| } |
| out.println(line); |
| } |
| catch (IOException e) { |
| break LOOP; |
| } |
| } |
| in.close(); |
| out.close(); |
| } |
| |
| /** |
| * The replaceCommand method takes a command (a macro call without the |
| * leading marker "$$") and computes replacement text for it. |
| * |
| * Most of the commands are simply names of integer constants that are defined |
| * in the source code of this GenerateCharacter class. The replacement text is |
| * simply the value of the constant as an appropriately formatted integer literal. |
| * |
| * Two cases are more complicated, however. The command "Tables" causes the |
| * final map and compressed tables to be emitted, with elaborate comments |
| * describing their contents. (This is actually handled by method genTables.) |
| * The command "Lookup(xxx)", where "xxx" is the name of a variable, generates |
| * an expression that will return the character property data for the character |
| * whose code is the value of the variable "xxx". (this is handled by method |
| * "genAccess".) |
| * |
| * @param x a command from the template file to be replaced |
| * @return the replacement text, as a String |
| * |
| * @see GenerateCharacter#genTables |
| * @see GenerateCharacter#genAccess |
| * @see GenerateCharacter#generateCharacterClass |
| */ |
| |
| static String replaceCommand(String x) { |
| if (x.equals("Tables")) return genTables(); |
| if (x.equals("Initializers")) return genInitializers(); |
| if (x.length() >= 9 && x.substring(0, 7).equals("Lookup(") && |
| x.substring(x.length()-1).equals(")") ) |
| return genAccess("A", x.substring(7, x.length()-1), (identifiers ? 2 : 32)); |
| if (x.length() >= 11 && x.substring(0, 9).equals("LookupEx(") && |
| x.substring(x.length()-1).equals(")") ) |
| return genAccess("B", x.substring(9, x.length()-1), 16); |
| if (x.equals("shiftType")) return Long.toString(shiftType); |
| if (x.equals("shiftIdentifierInfo")) return Long.toString(shiftIdentifierInfo); |
| if (x.equals("maskIdentifierInfo")) return "0x" + hex8(maskIdentifierInfo); |
| if (x.equals("maskUnicodePart")) return "0x" + hex8(maskUnicodePart); |
| if (x.equals("shiftCaseOffset")) return Long.toString(shiftCaseOffset); |
| if (x.equals("shiftCaseInfo")) return Long.toString(shiftCaseInfo); |
| if (x.equals("shiftCaseOffsetSign")) return Long.toString(shiftCaseOffsetSign); |
| if (x.equals("maskCase")) return "0x" + hex8(maskCase); |
| if (x.equals("maskCaseOffset")) return "0x" + hex8(maskCaseOffset); |
| if (x.equals("maskLowerCase")) return "0x" + hex8(maskLowerCase); |
| if (x.equals("maskUpperCase")) return "0x" + hex8(maskUpperCase); |
| if (x.equals("maskTitleCase")) return "0x" + hex8(maskTitleCase); |
| if (x.equals("maskOtherLowercase")) return "0x" + hex4(maskOtherLowercase >> 32); |
| if (x.equals("maskOtherUppercase")) return "0x" + hex4(maskOtherUppercase >> 32); |
| if (x.equals("maskOtherAlphabetic")) return "0x" + hex4(maskOtherAlphabetic >> 32); |
| if (x.equals("maskIdeographic")) return "0x" + hex4(maskIdeographic >> 32); |
| if (x.equals("valueIgnorable")) return "0x" + hex8(valueIgnorable); |
| if (x.equals("valueJavaUnicodeStart")) return "0x" + hex8(valueJavaUnicodeStart); |
| if (x.equals("valueJavaOnlyStart")) return "0x" + hex8(valueJavaOnlyStart); |
| if (x.equals("valueJavaUnicodePart")) return "0x" + hex8(valueJavaUnicodePart); |
| if (x.equals("valueJavaOnlyPart")) return "0x" + hex8(valueJavaOnlyPart); |
| if (x.equals("valueJavaWhitespace")) return "0x" + hex8(valueJavaWhitespace); |
| if (x.equals("lowJavaStart")) return "0x" + hex8(lowJavaStart); |
| if (x.equals("nonzeroJavaPart")) return "0x" + hex8(nonzeroJavaPart); |
| if (x.equals("bitJavaStart")) return "0x" + hex8(bitJavaStart); |
| if (x.equals("bitJavaPart")) return Long.toString(bitJavaPart); |
| if (x.equals("valueUnicodeStart")) return "0x" + hex8(valueUnicodeStart); |
| if (x.equals("maskIsJavaIdentifierStart")) return "0x" + hex(maskIsJavaIdentifierStart); |
| if (x.equals("maskIsJavaIdentifierPart")) return "0x" + hex(maskIsJavaIdentifierPart); |
| if (x.equals("shiftDigitOffset")) return Long.toString(shiftDigitOffset); |
| if (x.equals("maskDigitOffset")) return "0x" + hex(maskDigitOffset); |
| if (x.equals("maskDigit")) return "0x" + hex(maskDigit); |
| if (x.equals("shiftNumericType")) return Long.toString(shiftNumericType); |
| if (x.equals("maskNumericType")) return "0x" + hex(maskNumericType); |
| if (x.equals("valueNotNumeric")) return "0x" + hex8(valueNotNumeric); |
| if (x.equals("valueDigit")) return "0x" + hex8(valueDigit); |
| if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric); |
| if (x.equals("valueJavaSupradecimal")) return "0x" + hex8(valueJavaSupradecimal); |
| if (x.equals("valueDigit")) return "0x" + hex8(valueDigit); |
| if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric); |
| if (x.equals("maskType")) return "0x" + hex(maskType); |
| if (x.equals("shiftBidi")) return Long.toString(shiftBidi); |
| if (x.equals("maskBidi")) return "0x" + hex(maskBidi); |
| if (x.equals("maskMirrored")) return "0x" + hex8(maskMirrored); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UNASSIGNED][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.UNASSIGNED); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UPPERCASE_LETTER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.UPPERCASE_LETTER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LOWERCASE_LETTER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.LOWERCASE_LETTER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.TITLECASE_LETTER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.TITLECASE_LETTER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_LETTER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.MODIFIER_LETTER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_LETTER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.OTHER_LETTER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.NON_SPACING_MARK][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.NON_SPACING_MARK); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.ENCLOSING_MARK][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.ENCLOSING_MARK); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.COMBINING_SPACING_MARK][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.COMBINING_SPACING_MARK); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DECIMAL_DIGIT_NUMBER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DECIMAL_DIGIT_NUMBER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_NUMBER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.OTHER_NUMBER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SPACE_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.SPACE_SEPARATOR); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LINE_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.LINE_SEPARATOR); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PARAGRAPH_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.PARAGRAPH_SEPARATOR); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONTROL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.CONTROL); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FORMAT][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.FORMAT); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PRIVATE_USE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.PRIVATE_USE); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SURROGATE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.SURROGATE); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DASH_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DASH_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.START_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.START_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.END_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.END_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.INITIAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.INITIAL_QUOTE_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FINAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.FINAL_QUOTE_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONNECTOR_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.CONNECTOR_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_PUNCTUATION][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.OTHER_PUNCTUATION); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LETTER_NUMBER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.LETTER_NUMBER); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MATH_SYMBOL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.MATH_SYMBOL); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CURRENCY_SYMBOL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.CURRENCY_SYMBOL); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_SYMBOL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.MODIFIER_SYMBOL); |
| if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_SYMBOL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.OTHER_SYMBOL); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_WHITESPACE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_WHITESPACE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_ISOLATE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_ISOLATE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ISOLATE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ISOLATE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_FIRST_STRONG_ISOLATE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_FIRST_STRONG_ISOLATE); |
| if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_ISOLATE][UnicodeSpec.LONG])) |
| return Integer.toString(UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_ISOLATE); |
| FAIL("Unknown text substitution marker " + commandMarker + x); |
| return commandMarker + x; |
| } |
| |
| /** |
| * The genTables method generates source code for all the lookup tables |
| * needed to represent the various Unicode character properties. |
| * It simply calls the method genTable once for each table to be generated |
| * and then generates a summary comment. |
| * |
| * @return the replacement text for the "Tables" command, as a String |
| * |
| * @see GenerateCharacter#genTable |
| * @see GenerateCharacter#replaceCommand |
| */ |
| static String genTables() { |
| int n = sizes.length; |
| StringBuffer result = new StringBuffer(); |
| // liu : Add a comment showing the source of this table |
| result.append(commentStart + " The following tables and code generated using:" + |
| commentEnd + "\n "); |
| result.append(commentStart + ' ' + commandLineDescription + commentEnd + "\n "); |
| |
| if (plane == 0 && bLatin1 == false) { |
| genCaseMapTableDeclaration(result); |
| genCaseMapTable(initializers, specialCaseMaps); |
| } |
| int totalBytes = 0; |
| for (int k = 0; k < n - 1; k++) { |
| genTable(result, tableNames[k], tables[k], 0, bytes[k]<<3, sizes[k], preshifted[k], |
| sizes[k+1], false, false, k==0); |
| int s = bytes[k]; |
| if (s == 1 && useCharForByte) { |
| s = 2; |
| } |
| totalBytes += tables[k].length * s; |
| } |
| genTable(result, "A", tables[n - 1], 0, (identifiers ? 2 : 32), |
| sizes[n - 1], false, 0, true, !(identifiers), false); |
| |
| // If we ever need more than 32 bits to represent the character properties, |
| // then a table "B" may be needed as well. |
| genTable(result, "B", tables[n - 1], 32, 16, sizes[n - 1], false, 0, true, true, false); |
| |
| totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32)) + 31) >> 5) << 2); |
| result.append(commentStart); |
| result.append(" In all, the character property tables require "); |
| result.append(totalBytes).append(" bytes.").append(commentEnd); |
| if (verbose) { |
| System.out.println("The character property tables require " |
| + totalBytes + " bytes."); |
| } |
| return result.toString(); |
| } |
| |
| /** |
| * The genInitializers method generates the body of the |
| * ensureInitted() method, which enables lazy initialization of |
| * the case map table and other tables. |
| */ |
| static String genInitializers() { |
| return initializers.toString(); |
| } |
| |
| /** |
| * Return the total number of bytes needed by all tables. This is a stripped- |
| * down copy of genTables(). |
| */ |
| static int getTotalBytes() { |
| int n = sizes.length; |
| int totalBytes = 0; |
| for (int k = 0; k < n - 1; k++) { |
| totalBytes += tables[k].length * bytes[k]; |
| } |
| totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32)) |
| + 31) >> 5) << 2); |
| return totalBytes; |
| } |
| |
| static void appendEscapedStringFragment(StringBuffer result, |
| char[] line, |
| int length, |
| boolean lastFragment) { |
| result.append(" \""); |
| for (int k=0; k<length; ++k) { |
| result.append("\\u"); |
| result.append(hex4(line[k])); |
| } |
| result.append("\""); |
| result.append(lastFragment ? ";" : "+"); |
| result.append("\n"); |
| } |
| |
| static String SMALL_INITIALIZER = |
| " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+ |
| // " $$name = new $$type[$$size];\n"+ |
| " int len = $$name_DATA.length();\n"+ |
| " int j=0;\n"+ |
| " for (int i=0; i<len; ++i) {\n"+ |
| " int c = $$name_DATA.charAt(i);\n"+ |
| " for (int k=0; k<$$entriesPerChar; ++k) {\n"+ |
| " $$name[j++] = ($$type)c;\n"+ |
| " c >>= $$bits;\n"+ |
| " }\n"+ |
| " }\n"+ |
| " assert (j == $$size);\n"+ |
| " }\n"; |
| |
| static String SAME_SIZE_INITIALIZER = |
| " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+ |
| " assert ($$name_DATA.length() == $$size);\n"+ |
| // " $$name = new $$type[$$size];\n"+ |
| " for (int i=0; i<$$size; ++i)\n"+ |
| " $$name[i] = ($$type)$$name_DATA.charAt(i);\n"+ |
| " }\n"; |
| |
| static String BIG_INITIALIZER = |
| " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+ |
| // " $$name = new $$type[$$size];\n"+ |
| " int len = $$name_DATA.length();\n"+ |
| " int j=0;\n"+ |
| " int charsInEntry=0;\n"+ |
| " $$type entry=0;\n"+ |
| " for (int i=0; i<len; ++i) {\n"+ |
| " entry |= $$name_DATA.charAt(i);\n"+ |
| " if (++charsInEntry == $$charsPerEntry) {\n"+ |
| " $$name[j++] = entry;\n"+ |
| " entry = 0;\n"+ |
| " charsInEntry = 0;\n"+ |
| " }\n"+ |
| " else {\n"+ |
| " entry <<= 16;\n"+ |
| " }\n"+ |
| " }\n"+ |
| " assert (j == $$size);\n"+ |
| " }\n"; |
| |
| static String INT32_INITIALIZER = |
| " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+ |
| " char[] data = $$name_DATA.toCharArray();\n"+ |
| " assert (data.length == ($$size * 2));\n"+ |
| " int i = 0, j = 0;\n"+ |
| " while (i < ($$size * 2)) {\n"+ |
| " int entry = data[i++] << 16;\n"+ |
| " $$name[j++] = entry | data[i++];\n"+ |
| " }\n"+ |
| " }\n"; |
| |
| static void addInitializer(String name, String type, int entriesPerChar, |
| int bits, int size) { |
| |
| String template = (entriesPerChar == 1) ? SAME_SIZE_INITIALIZER : |
| ((entriesPerChar > 0) ? SMALL_INITIALIZER : BIG_INITIALIZER); |
| if (entriesPerChar == -2) { |
| template = INT32_INITIALIZER; |
| } |
| int marklen = commandMarker.length(); |
| int pos = 0; |
| while ((pos = template.indexOf(commandMarker, pos)) >= 0) { |
| int newpos = pos + marklen; |
| char ch = 'x'; |
| while (newpos < template.length() && |
| Character.isJavaIdentifierStart(ch = template.charAt(newpos)) && |
| ch != '_') // Don't allow this in token names |
| ++newpos; |
| String token = template.substring(pos+marklen, newpos); |
| String replacement = "ERROR"; |
| |
| if (token.equals("name")) replacement = name; |
| else if (token.equals("type")) replacement = type; |
| else if (token.equals("bits")) replacement = ""+bits; |
| else if (token.equals("size")) replacement = ""+size; |
| else if (token.equals("entriesPerChar")) replacement = ""+entriesPerChar; |
| else if (token.equals("charsPerEntry")) replacement = ""+(-entriesPerChar); |
| else FAIL("Unrecognized token: " + token); |
| |
| template = template.substring(0, pos) + replacement + template.substring(newpos); |
| pos += replacement.length(); |
| } |
| initializers.append(template); |
| } |
| |
| /** |
| * The genTable method generates source code for one lookup table. |
| * Most of the complexity stems from handling various options as to |
| * the type of the array components, the precise representation of the |
| * values, the format in which to render each value, the number of values |
| * to emit on each line of source code, and the kinds of useful comments |
| * to be generated. |
| * |
| * @param result a StringBuffer, to which the generated source code |
| * text is to be appended |
| * @param name the name of the table |
| * @param table the table data (an array of long values) |
| * @param extract a distance, in bits, by which each entry of the table |
| * is to be right-shifted before it is processed |
| * @param bits the number of bits (not bytes) to be used to represent |
| * each table entry |
| * @param size the table data is divided up into blocks of size (1<<size); |
| * in this method, this information is used only to affect |
| * how many table values are to be generated per line |
| * @param preshifted if this flag is true, then the table entries are to be |
| * emitted in a preshifted form; that is, each value should |
| * be left-shifted by the amount "shift", so that this work |
| * is built into the table and need not be performed by an |
| * explicit shift operator at run time |
| * @param shift this is the shift amount for preshifting of table entries |
| * @param hexFormat if this flag is true, table entries should be emitted as |
| * hexadecimal literals; otherwise decimal literals are used |
| * @param properties if this flag is true, the table entries are encoded |
| * character properties rather than indexes into yet other tables; |
| * therefore comments describing the encoded properties should |
| * be generated |
| * @param hexComment if this flag is true, each line of output is labelled with |
| * a hexadecimal comment indicating the character values to |
| * which that line applies; otherwise, decimal values indicating |
| * table indices are generated |
| * |
| * @see GenerateCharacter#genTables |
| * @see GenerateCharacter#replaceCommand |
| */ |
| |
| static void genTable(StringBuffer result, String name, |
| long[] table, int extract, int bits, int size, |
| boolean preshifted, int shift, boolean hexFormat, |
| boolean properties, boolean hexComment) { |
| |
| String atype = bits == 1 ? (Csyntax ? "unsigned long" : "int") : |
| bits == 2 ? (Csyntax ? "unsigned long" : "int") : |
| bits == 4 ? (Csyntax ? "unsigned long" : "int") : |
| bits == 8 ? (Csyntax ? "unsigned char" : "byte") : |
| bits == 16 ? (Csyntax ? "unsigned short" : "char") : |
| bits == 32 ? (Csyntax ? "unsigned long" : "int") : |
| (Csyntax ? "int64" : "long"); |
| long maxPosEntry = bits == 1 ? Integer.MAX_VALUE : // liu |
| bits == 2 ? Integer.MAX_VALUE : |
| bits == 4 ? Integer.MAX_VALUE : |
| bits == 8 ? Byte.MAX_VALUE : |
| bits == 16 ? Short.MAX_VALUE : |
| bits == 32 ? Integer.MAX_VALUE : |
| Long.MAX_VALUE; |
| int entriesPerChar = bits <= 16 ? (16 / bits) : -(bits / 16); |
| boolean shiftEntries = preshifted && shift != 0; |
| if (bits == 8 && tableAsString && useCharForByte) { |
| atype = "char"; |
| maxPosEntry = Character.MAX_VALUE; |
| entriesPerChar = 1; |
| } |
| boolean noConversion = atype.equals("char"); |
| |
| result.append(commentStart); |
| result.append(" The ").append(name).append(" table has ").append(table.length); |
| result.append(" entries for a total of "); |
| int sizeOfTable = ((table.length * bits + 31) >> 5) << 2; |
| if (bits == 8 && useCharForByte) { |
| sizeOfTable *= 2; |
| } |
| result.append(sizeOfTable); |
| result.append(" bytes.").append(commentEnd).append("\n\n"); |
| if (Csyntax) |
| result.append(" static "); |
| else |
| result.append(" static final "); |
| result.append(atype); |
| result.append(" ").append(name).append("["); |
| if (Csyntax) |
| result.append(table.length >> (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0)); |
| if (tableAsString) { |
| if (noConversion) { |
| result.append("] = (\n"); |
| } else { |
| result.append("] = new ").append(atype).append("["+table.length+"];\n "); |
| result.append("static final String ").append(name).append("_DATA =\n"); |
| } |
| int CHARS_PER_LINE = 8; |
| StringBuffer theString = new StringBuffer(); |
| int entriesInCharSoFar = 0; |
| char ch = '\u0000'; |
| int charsPerEntry = -entriesPerChar; |
| for (int j=0; j<table.length; ++j) { |
| //long entry = table[j] >> extract; |
| long entry; |
| if ("A".equals(name)) |
| entry = (table[j] & 0xffffffffL) >> extract; |
| else |
| entry = (table[j] >> extract); |
| if (shiftEntries) entry <<= shift; |
| if (entry >= (1L << bits)) { |
| FAIL("Entry too big"); |
| } |
| if (entriesPerChar > 0) { |
| // Pack multiple entries into a character |
| ch = (char)(((int)ch >> bits) | (entry << (entriesPerChar-1)*bits)); |
| ++entriesInCharSoFar; |
| if (entriesInCharSoFar == entriesPerChar) { |
| // Character is full |
| theString.append(ch); |
| entriesInCharSoFar = 0; |
| ch = '\u0000'; |
| } |
| } |
| else { |
| // Use multiple characters per entry |
| for (int k=0; k<charsPerEntry; ++k) { |
| ch = (char)(entry >> ((charsPerEntry-1)*16)); |
| entry <<= 16; |
| theString.append(ch); |
| } |
| } |
| } |
| if (entriesInCharSoFar > 0) { |
| while (entriesInCharSoFar < entriesPerChar) { |
| ch = (char)((int)ch >> bits); |
| ++entriesInCharSoFar; |
| } |
| theString.append(ch); |
| entriesInCharSoFar = 0; |
| } |
| result.append(Utility.formatForSource(theString.toString(), " ")); |
| if (noConversion) { |
| result.append(").toCharArray()"); |
| } |
| result.append(";\n\n "); |
| |
| if (!noConversion) { |
| addInitializer(name, atype, entriesPerChar, bits, table.length); |
| } |
| } |
| else { |
| result.append("] = {"); |
| boolean castEntries = shiftEntries && (bits < 32); |
| int printPerLine = hexFormat ? (bits == 1 ? 32*4 : |
| bits == 2 ? 16*4 : |
| bits == 4 ? 8*4 : |
| bits == 8 ? 8 : |
| bits == 16 ? 8 : |
| bits == 32 ? 4 : 2) : |
| (bits == 8 ? 8 : |
| bits == 16 ? 8 : 4); |
| int printMask = properties ? 0 : |
| Math.min(1 << size, |
| printPerLine >> (castEntries ? (Csyntax ? 2 : 1) : 0)) - 1; |
| int commentShift = ((1 << size) == table.length) ? 0 : size; |
| int commentMask = ((1 << size) == table.length) ? printMask : (1 << size) - 1; |
| long val = 0; |
| for (int j = 0; j < table.length; j++) { |
| if ((j & printMask) == 0) { |
| while (result.charAt(result.length() - 1) == ' ') |
| result.setLength(result.length() - 1); |
| result.append("\n "); |
| } |
| PRINT: { |
| if (castEntries) |
| result.append("(").append(atype).append(")("); |
| long entry = table[j] >> extract; |
| int packMask = ((1 << (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 2)) - 1); |
| int k = j & packMask; |
| if (bits >= 8) |
| val = entry; |
| else if (k == 0) { |
| val = entry; |
| break PRINT; |
| } |
| else { |
| val |= (entry << (k*bits)); |
| if (k != packMask) |
| break PRINT; |
| } |
| if (val > maxPosEntry && !Csyntax) { // liu |
| // For values that are out of range, convert them to in-range negative values. |
| // Actually, output the '-' and convert them to the negative of the corresponding |
| // in-range negative values. E.g., convert 130 == -126 (in 8 bits) -> 126. |
| result.append('-'); |
| val = maxPosEntry + maxPosEntry + 2 - val; |
| } |
| if (hexFormat) { |
| result.append("0x"); |
| if (bits == 8) |
| result.append(hex2((byte)val)); |
| else if (bits == 16) |
| result.append(hex4((short)val)); |
| else if (bits == 32 || bits < 8) |
| result.append(hex8((int)val)); |
| else { |
| result.append(hex16(val)); |
| if (!Csyntax) |
| result.append("L"); |
| } |
| } |
| else { |
| if (bits == 8) |
| result.append(dec3(val)); |
| else if (bits == 64) { |
| result.append(dec5(val)); |
| if (!Csyntax) |
| result.append("L"); |
| } |
| else |
| result.append(dec5(val)); |
| } |
| if (shiftEntries) |
| result.append("<<").append(shift); |
| if (castEntries) result.append(")"); |
| if (j < (table.length - 1)) |
| result.append(", "); |
| else |
| result.append(" "); |
| if ((j & printMask) == printMask) { |
| result.append(" ").append(commentStart).append(" "); |
| if (hexComment) |
| result.append("0x").append(hex4((j & ~commentMask) << (16 - size))); |
| else |
| result.append(dec3((j & ~commentMask) >> commentShift)); |
| if (properties) propertiesComments(result, val); |
| result.append(commentEnd); |
| } |
| } // end PRINT |
| } |
| result.append("\n };\n\n "); |
| } |
| } |
| |
| static void genCaseMapTableDeclaration(StringBuffer result) { |
| String myTab = " "; |
| result.append(myTab + "static final char[][][] charMap;\n"); |
| } |
| |
| static void genCaseMapTable(StringBuffer result, SpecialCaseMap[] specialCaseMaps){ |
| String myTab = " "; |
| int ch; |
| char[] map; |
| result.append(myTab + "charMap = new char[][][] {\n"); |
| for (int x = 0; x < specialCaseMaps.length; x++) { |
| ch = specialCaseMaps[x].getCharSource(); |
| map = specialCaseMaps[x].getUpperCaseMap(); |
| result.append(myTab + myTab); |
| result.append("{ "); |
| result.append("{\'\\u"+hex4(ch)+"\'}, {"); |
| for (int y = 0; y < map.length; y++) { |
| result.append("\'\\u"+hex4(map[y])+"\', "); |
| } |
| result.append("} },\n"); |
| } |
| result.append(myTab + "};\n"); |
| |
| } |
| |
| /** |
| * The propertiesComments method generates comments describing encoded |
| * character properties. |
| * |
| * @param result a StringBuffer, to which the generated source code |
| * text is to be appended |
| * @param val encoded character properties |
| * |
| * @see GenerateCharacter#genTable |
| */ |
| |
| static void propertiesComments(StringBuffer result, long val) { |
| result.append(" "); |
| switch ((int)(val & maskType)) { |
| case UnicodeSpec.CONTROL: |
| result.append("Cc"); |
| break; |
| case UnicodeSpec.FORMAT: |
| result.append("Cf"); |
| break; |
| case UnicodeSpec.PRIVATE_USE: |
| result.append("Co"); |
| break; |
| case UnicodeSpec.SURROGATE: |
| result.append("Cs"); |
| break; |
| case UnicodeSpec.LOWERCASE_LETTER: |
| result.append("Ll"); |
| break; |
| case UnicodeSpec.MODIFIER_LETTER: |
| result.append("Lm"); |
| break; |
| case UnicodeSpec.OTHER_LETTER: |
| result.append("Lo"); |
| break; |
| case UnicodeSpec.TITLECASE_LETTER: |
| result.append("Lt"); |
| break; |
| case UnicodeSpec.UPPERCASE_LETTER: |
| result.append("Lu"); |
| break; |
| case UnicodeSpec.COMBINING_SPACING_MARK: |
| result.append("Mc"); |
| break; |
| case UnicodeSpec.ENCLOSING_MARK: |
| result.append("Me"); |
| break; |
| case UnicodeSpec.NON_SPACING_MARK: |
| result.append("Mn"); |
| break; |
| case UnicodeSpec.DECIMAL_DIGIT_NUMBER: |
| result.append("Nd"); |
| break; |
| case UnicodeSpec.LETTER_NUMBER: |
| result.append("Nl"); |
| break; |
| case UnicodeSpec.OTHER_NUMBER: |
| result.append("No"); |
| break; |
| case UnicodeSpec.CONNECTOR_PUNCTUATION: |
| result.append("Pc"); |
| break; |
| case UnicodeSpec.DASH_PUNCTUATION: |
| result.append("Pd"); |
| break; |
| case UnicodeSpec.END_PUNCTUATION: |
| result.append("Pe"); |
| break; |
| case UnicodeSpec.OTHER_PUNCTUATION: |
| result.append("Po"); |
| break; |
| case UnicodeSpec.START_PUNCTUATION: |
| result.append("Ps"); |
| break; |
| case UnicodeSpec.CURRENCY_SYMBOL: |
| result.append("Sc"); |
| break; |
| case UnicodeSpec.MODIFIER_SYMBOL: |
| result.append("Sk"); |
| break; |
| case UnicodeSpec.MATH_SYMBOL: |
| result.append("Sm"); |
| break; |
| case UnicodeSpec.OTHER_SYMBOL: |
| result.append("So"); |
| break; |
| case UnicodeSpec.LINE_SEPARATOR: |
| result.append("Zl"); break; |
| case UnicodeSpec.PARAGRAPH_SEPARATOR: |
| result.append("Zp"); |
| break; |
| case UnicodeSpec.SPACE_SEPARATOR: |
| result.append("Zs"); |
| break; |
| case UnicodeSpec.UNASSIGNED: |
| result.append("unassigned"); |
| break; |
| } |
| |
| switch ((int)((val & maskBidi) >> shiftBidi)) { |
| case UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT: |
| result.append(", L"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT: |
| result.append(", R"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER: |
| result.append(", EN"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR: |
| result.append(", ES"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR: |
| result.append(", ET"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER: |
| result.append(", AN"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR: |
| result.append(", CS"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR: |
| result.append(", B"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR: |
| result.append(", S"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_WHITESPACE: |
| result.append(", WS"); |
| break; |
| case UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS: |
| result.append(", ON"); |
| break; |
| } |
| if ((val & maskUpperCase) != 0) { |
| result.append(", hasUpper (subtract "); |
| result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")"); |
| } |
| if ((val & maskLowerCase) != 0) { |
| result.append(", hasLower (add "); |
| result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")"); |
| } |
| if ((val & maskTitleCase) != 0) { |
| result.append(", hasTitle"); |
| } |
| if ((val & maskIdentifierInfo) == valueIgnorable) { |
| result.append(", ignorable"); |
| } |
| if ((val & maskIdentifierInfo) == valueJavaUnicodePart) { |
| result.append(", identifier part"); |
| } |
| if ((val & maskIdentifierInfo) == valueJavaStartUnicodePart) { |
| result.append(", underscore"); |
| } |
| if ((val & maskIdentifierInfo) == valueJavaWhitespace) { |
| result.append(", whitespace"); |
| } |
| if ((val & maskIdentifierInfo) == valueJavaOnlyStart) { |
| result.append(", currency"); |
| } |
| if ((val & maskIdentifierInfo) == valueJavaUnicodeStart) { |
| result.append(", identifier start"); |
| } |
| if ((val & maskNumericType) == valueDigit) { |
| result.append(", decimal "); |
| result.append((val & maskDigitOffset) >> shiftDigitOffset); |
| } |
| if ((val & maskNumericType) == valueStrangeNumeric) { |
| result.append(", strange"); |
| } |
| if ((val & maskNumericType) == valueJavaSupradecimal) { |
| result.append(", supradecimal "); |
| result.append((val & maskDigitOffset) >> shiftDigitOffset); |
| } |
| } |
| |
| static String[] tableNames = { "X", "Y", "Z", "P", "Q", "R", "S", "T", "U", "V", "W" }; |
| |
| static String tableName(int j) { return tableNames[j]; } |
| |
| /** |
| * The genAccess method generates source code for one table access expression. |
| * |
| * Most of the complexity stems from handling various options as to |
| * table representation, such as whether it contains values so large that |
| * they are represented as negative values and whether the table values are |
| * preshifted. This method also avoids such "ugly" expressions as shifting |
| * by distance zero, masking when no masking is necessary, and so on. |
| * For clarity, it generates expressions that do not rely on operator |
| * precedence, but otherwise it avoids generating redundant parentheses. |
| * |
| * A generated expression might look like A[Y[(X[ch>>6]<<6)|(ch&0x3F)]] |
| * or A[Z[Y[(X[ch>>7]<<4)|((ch>>3)&0xF)]|(ch&0x7)]], for example. |
| * |
| * @param tbl the name of the final table to be accessed |
| * @param var the variable name that appeared in parentheses in the |
| * "Lookup" command |
| * @param bits the number of bits (not bytes) to be used to represent |
| * the final table entry |
| * @return the replacement text for the "Lookup(xxx)" command, as a String |
| * |
| * @see GenerateCharacter#replaceCommand |
| */ |
| |
| static String genAccess(String tbl, String var, int bits) { |
| String access = null; |
| int bitoffset = bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0; |
| for (int k = 0; k < sizes.length; k++) { |
| int offset = ((k < sizes.length - 1) ? 0 : bitoffset); |
| int shift = shifts[k] + offset; |
| String shifted = (shift == 0) ? var : "(" + var + ">>" + shift + ")"; |
| int mask = (1 << (sizes[k] - offset)) - 1; |
| String masked = (k == 0) ? shifted : |
| "(" + shifted + "&0x" + hex(mask) + ")"; |
| String index = (k == 0) ? masked : |
| (mask == 0) ? access : "(" + access + "|" + masked + ")"; |
| String indexNoParens = (index.charAt(0) != '(') ? index : |
| index.substring(1, index.length() - 1); |
| String tblname = (k == sizes.length - 1) ? tbl : tableName(k); |
| String fetched = tblname + "[" + indexNoParens + "]"; |
| String zeroextended = (zeroextend[k] == 0) ? fetched : |
| "(" + fetched + "&0x" + hex(zeroextend[k]) + ")"; |
| int adjustment = preshifted[k] ? 0 : |
| sizes[k+1] - ((k == sizes.length - 2) ? bitoffset : 0); |
| String adjusted = (preshifted[k] || adjustment == 0) ? zeroextended : |
| "(" + zeroextended + "<<" + adjustment + ")"; |
| String bitshift = (bits == 1) ? "(" + var + "&0x1F)" : |
| (bits == 2) ? "((" + var + "&0xF)<<1)" : |
| (bits == 4) ? "((" + var + "&7)<<2)" : null; |
| String extracted = ((k < sizes.length - 1) || (bits >= 8)) ? adjusted : |
| "((" + adjusted + ">>" + bitshift + ")&" + |
| (bits == 4 ? "0xF" : "" + ((1 << bits) - 1)) + ")"; |
| access = extracted; |
| } |
| return access; |
| } |
| |
| /* The command line arguments are decoded and used to set the following |
| global variables. |
| */ |
| |
| static boolean verbose = false; |
| static boolean nobidi = false; |
| static boolean nomirror = false; |
| static boolean identifiers = false; |
| static boolean Csyntax = false; |
| static String TemplateFileName = null; |
| static String OutputFileName = null; |
| static String UnicodeSpecFileName = null; // liu |
| static String SpecialCasingFileName = null; |
| static String PropListFileName = null; |
| static boolean useCharForByte = false; |
| static int[] sizes; |
| static int bins = 0; // liu; if > 0, then perform search |
| static boolean tableAsString = false; |
| static boolean bLatin1 = false; |
| |
| static String commandLineDescription; |
| |
| /* Other global variables, equal in length to the "sizes" array. */ |
| |
| static int[] shifts; |
| static int[] zeroextend; |
| static int[] bytes; |
| static boolean[] preshifted; |
| static long[][] tables; |
| |
| |
| /* Other global variables */ |
| static String commentStart; |
| static String commentEnd; |
| |
| static StringBuffer initializers = new StringBuffer(); |
| |
| /* special casing rules for 1:M toUpperCase mappings */ |
| static SpecialCaseMap[] specialCaseMaps; |
| |
| /** |
| * Process the command line arguments. |
| * |
| * The allowed flags in command line are: |
| * <dl> |
| * <dt> -verbose <dd> Emit comments to standard output describing |
| * what's going on during the processing. |
| * <dt> -nobidi <dd> Do not include bidi categories in the |
| * encoded character properties. |
| * <dt> -nomirror <dd> Do no include mirror property in the encoded |
| * character properties. |
| * <dt> -identifiers <dd> Generate tables for scanning identifiers only. |
| * <dt> -c <dd> Output code in C syntax instead of Java syntax. |
| * <dt> -o filename <dd> Specify output file name. |
| * <dt> -template filename <dd> Specify template input file name. |
| * <dt> -spec filename <dd> Specify Unicode spec file name. |
| * <dt> -specialcasing filename <dd> Specify Unicode special casing file name. |
| * <dt> -search bins <dd> Try different partitions into the specified |
| * number of bins. E.g., for 2 bins, try |
| * 16 0, 15 1,..., 0 16. |
| * <dt> -string <dd> Create table as string. Only valid with Java |
| * syntax. |
| * <dt> -latin1 <dd> Create a latin 1 only property table. |
| * </dl> |
| * In addition, decimal literals may appear as command line arguments; |
| * each one represents the number of bits of the character to be broken |
| * off at each lookup step. If present, they must add up to 16 (the number |
| * of bits in a char value). For smaller tables, the last value should |
| * be 0; values other than the last one may not be zero. If no such |
| * numeric values are provided, default values are used. |
| * |
| * @param args the command line arguments, as an array of String |
| * |
| * @see GenerateCharacter#main |
| */ |
| |
| static void processArgs(String[] args) { |
| StringBuffer desc = new StringBuffer("java GenerateCharacter"); |
| for (int j=0; j<args.length; ++j) { |
| desc.append(" " + args[j]); |
| } |
| for (int j = 0; j < args.length; j++) { |
| if (args[j].equals("-verbose") || args[j].equals("-v")) |
| verbose = true; |
| else if (args[j].equals("-nobidi")) |
| nobidi = true; |
| else if (args[j].equals("-nomirror")) |
| nomirror = true; |
| else if (args[j].equals("-identifiers")) |
| identifiers = true; |
| else if (args[j].equals("-c")) |
| Csyntax = true; |
| else if (args[j].equals("-string")) |
| tableAsString = true; |
| else if (args[j].equals("-o")) { |
| if (j == args.length - 1) { |
| FAIL("File name missing after -o"); |
| } |
| else { |
| OutputFileName = args[++j]; |
| } |
| } |
| else if (args[j].equals("-search")) { |
| if (j == args.length - 1) |
| FAIL("Bin count missing after -search"); |
| else { |
| bins = Integer.parseInt(args[++j]); |
| if (bins < 1 || bins > 10) |
| FAIL("Bin count must be >= 1 and <= 10"); |
| } |
| } |
| else if (args[j].equals("-template")) { |
| if (j == args.length - 1) |
| FAIL("File name missing after -template"); |
| else |
| TemplateFileName = args[++j]; |
| } |
| else if (args[j].equals("-spec")) { // liu |
| if (j == args.length - 1) { |
| FAIL("File name missing after -spec"); |
| } |
| else { |
| UnicodeSpecFileName = args[++j]; |
| } |
| } |
| else if (args[j].equals("-specialcasing")) { |
| if (j == args.length -1) { |
| FAIL("File name missing after -specialcasing"); |
| } |
| else { |
| SpecialCasingFileName = args[++j]; |
| } |
| } |
| else if (args[j].equals("-proplist")) { |
| if (j == args.length -1) { |
| FAIL("File name missing after -proplist"); |
| } |
| else { |
| PropListFileName = args[++j]; |
| } |
| } |
| else if (args[j].equals("-plane")) { |
| if (j == args.length -1) { |
| FAIL("Plane number missing after -plane"); |
| } |
| else { |
| plane = Integer.parseInt(args[++j]); |
| } |
| if (plane > 0) { |
| bLatin1 = false; |
| } |
| } |
| else if ("-usecharforbyte".equals(args[j])) { |
| useCharForByte = true; |
| } |
| else if (args[j].equals("-latin1")) { |
| bLatin1 = true; |
| plane = 0; |
| } |
| else { |
| try { |
| int val = Integer.parseInt(args[j]); |
| if (val < 0 || val > 32) FAIL("Incorrect bit field width: " + args[j]); |
| if (sizes == null) |
| sizes = new int[1]; |
| else { |
| int[] newsizes = new int[sizes.length + 1]; |
| System.arraycopy(sizes, 0, newsizes, 0, sizes.length); |
| sizes = newsizes; |
| } |
| sizes[sizes.length - 1] = val; |
| } |
| catch(NumberFormatException e) { |
| FAIL("Unknown switch: " + args[j]); |
| } |
| } |
| } |
| if (Csyntax && tableAsString) { |
| FAIL("Can't specify table as string with C syntax"); |
| } |
| if (sizes == null) { |
| desc.append(" ["); |
| if (identifiers) { |
| int[] newsizes = { 8, 4, 4 }; // Good default values |
| desc.append("8 4 4]"); |
| sizes = newsizes; |
| } |
| else { |
| int[] newsizes = { 10, 5, 1 }; // Guy's old defaults for 2.0.14: { 9, 4, 3, 0 } |
| desc.append("10 5 1]"); |
| sizes = newsizes; |
| } |
| } |
| if (UnicodeSpecFileName == null) { // liu |
| UnicodeSpecFileName = DefaultUnicodeSpecFileName; |
| desc.append(" [-spec " + UnicodeSpecFileName + ']'); |
| } |
| if (SpecialCasingFileName == null) { |
| SpecialCasingFileName = DefaultSpecialCasingFileName; |
| desc.append(" [-specialcasing " + SpecialCasingFileName + ']'); |
| } |
| if (PropListFileName == null) { |
| PropListFileName = DefaultPropListFileName; |
| desc.append(" [-proplist " + PropListFileName + ']'); |
| } |
| if (TemplateFileName == null) { |
| TemplateFileName = (Csyntax ? DefaultCTemplateFileName |
| : DefaultJavaTemplateFileName); |
| desc.append(" [-template " + TemplateFileName + ']'); |
| } |
| if (OutputFileName == null) { |
| OutputFileName = (Csyntax ? DefaultCOutputFileName |
| : DefaultJavaOutputFileName); |
| desc.append(" [-o " + OutputFileName + ']'); |
| } |
| commentStart = (Csyntax ? "/*" : "//"); |
| commentEnd = (Csyntax ? " */" : ""); |
| commandLineDescription = desc.toString(); |
| } |
| |
| private static void searchBins(long[] map, int binsOccupied) throws Exception { |
| int bitsFree = 16; |
| for (int i=0; i<binsOccupied; ++i) bitsFree -= sizes[i]; |
| if (binsOccupied == (bins-1)) { |
| sizes[binsOccupied] = bitsFree; |
| generateForSizes(map); |
| } |
| else { |
| for (int i=1; i<bitsFree; ++i) { // Don't allow bins of 0 except for last one |
| sizes[binsOccupied] = i; |
| searchBins(map, binsOccupied+1); |
| } |
| } |
| } |
| |
| private static void generateForSizes(long[] map) throws Exception { |
| int sum = 0; |
| shifts = new int[sizes.length]; |
| for (int k = sizes.length - 1; k >= 0; k--) { |
| shifts[k] = sum; |
| sum += sizes[k]; |
| } |
| if ((1 << sum) < map.length || (1 << (sum - 1)) >= map.length) { |
| FAIL("Bit field widths total to " + sum + |
| ": wrong total for map of size " + map.length); |
| } |
| // need a table for each set of lookup bits in char |
| tables = new long[sizes.length][]; |
| // the last table is the map |
| tables[sizes.length - 1] = map; |
| for (int j = sizes.length - 1; j > 0; j--) { |
| if (verbose && bins==0) |
| System.err.println("Building map " + (j+1) + " of bit width " + sizes[j]); |
| long[][] temp = buildTable(tables[j], sizes[j]); |
| tables[j-1] = temp[0]; |
| tables[j] = temp[1]; |
| } |
| preshifted = new boolean[sizes.length]; |
| zeroextend = new int[sizes.length]; |
| bytes = new int[sizes.length]; |
| for (int j = 0; j < sizes.length - 1; j++) { |
| int len = tables[j+1].length; |
| int size = sizes[j+1]; |
| if (len > 0x100 && (len >> size) <= 0x100) { |
| len >>= size; |
| preshifted[j] = false; |
| } |
| else if (len > 0x10000 && (len >> size) <= 0x10000) { |
| len >>= size; |
| preshifted[j] = false; |
| } |
| else preshifted[j] = true; |
| if (Csyntax) |
| zeroextend[j] = 0; |
| else if (len > 0x7F && len <= 0xFF) { |
| if (!useCharForByte) { |
| zeroextend[j] = 0xFF; |
| } |
| } else if (len > 0x7FFF && len <= 0xFFFF) |
| zeroextend[j] = 0xFFFF; |
| else zeroextend[j] = 0; |
| if (len <= 0x100) bytes[j] = 1; |
| else if (len <= 0x10000) bytes[j] = 2; |
| else bytes[j] = 4; |
| } |
| preshifted[sizes.length - 1] = true; |
| zeroextend[sizes.length - 1] = 0; |
| bytes[sizes.length - 1] = 0; |
| if (bins > 0) { |
| int totalBytes = getTotalBytes(); |
| String access = genAccess("A", "ch", (identifiers ? 2 : 32)); |
| int accessComplexity = 0; |
| for (int j=0; j<access.length(); ++j) { |
| char ch = access.charAt(j); |
| if ("[&|><".indexOf(ch) >= 0) ++accessComplexity; |
| if (ch == '<' || ch == '>') ++j; |
| } |
| System.out.print("("); |
| for (int j=0; j<sizes.length; ++j) System.out.print(" " + sizes[j]); |
| System.out.println(" ) " + totalBytes + " " + accessComplexity + " " + access); |
| return; |
| } |
| if (verbose) { |
| System.out.println(" n\t size\tlength\tshift\tzeroext\tbytes\tpreshifted"); |
| for (int j = 0; j < sizes.length; j++) { |
| System.out.println(dec5(j) + "\t" + |
| dec5(sizes[j]) + "\t" + |
| dec5(tables[j].length) + "\t" + |
| dec5(shifts[j]) + "\t" + |
| dec5(zeroextend[j]) + "\t" + |
| dec5(bytes[j]) + "\t " + |
| preshifted[j]); |
| } |
| } |
| if (verbose) { |
| System.out.println("Generating source code for class Character"); |
| System.out.println("A table access looks like " + |
| genAccess("A", "ch", (identifiers ? 2 : 32))); |
| } |
| generateCharacterClass(TemplateFileName, OutputFileName); |
| } |
| |
| /** |
| * The main program for generating source code for the Character class. |
| * The basic outline of its operation is: |
| * <ol> |
| * <li> Process the command line arguments. One result of this process |
| * is a list of sizes (measured in bits and summing to 16). |
| * <li> Get the Unicode character property data from the specification file. |
| * <li> From that, build a map that has, for each character code, its |
| * relevant properties encoded as a long integer value. |
| * <li> Repeatedly compress the map, producing a compressed table and a |
| * new map. This is done once for each size value in the list. |
| * When this is done, we have a set of tables. |
| * <li> Make some decisions about table representation; record these |
| * decisions in arrays named preshifted, zeroextend, and bytes. |
| * <li> Generate the source code for the class Character by performing |
| * macro processing on a template file. |
| * </ol> |
| * |
| * @param args the command line arguments, as an array of String |
| * |
| * @see GenerateCharacter#processArgs |
| * @see UnicodeSpec@readSpecFile |
| * @see GenerateCharacter#buildMap |
| * @see GenerateCharacter#buildTable |
| * @see GenerateCharacter#generateCharacterClass |
| */ |
| |
| public static void main(String[] args) { |
| processArgs(args); |
| try { |
| |
| UnicodeSpec[] data = UnicodeSpec.readSpecFile(new File(UnicodeSpecFileName), plane); |
| specialCaseMaps = SpecialCaseMap.readSpecFile(new File(SpecialCasingFileName), plane); |
| PropList propList = PropList.readSpecFile(new File(PropListFileName), plane); |
| |
| if (verbose) { |
| System.out.println(data.length + " items read from Unicode spec file " + UnicodeSpecFileName); // liu |
| } |
| long[] map = buildMap(data, specialCaseMaps, propList); |
| if (verbose) { |
| System.err.println("Completed building of initial map"); |
| } |
| |
| if (bins == 0) { |
| generateForSizes(map); |
| } |
| else { |
| while (bins > 0) { |
| sizes = new int[bins]; |
| searchBins(map, 0); |
| --bins; |
| } |
| } |
| if (verbose && false) { |
| System.out.println("Offset range seen: -" + hex8(-minOffsetSeen) + "..+" + |
| hex8(maxOffsetSeen)); |
| System.out.println(" allowed: -" + hex8(-minOffset) + "..+" + |
| hex8(maxOffset)); |
| } |
| } |
| catch (FileNotFoundException e) { FAIL(e.toString()); } |
| catch (IOException e) { FAIL(e.toString()); } |
| catch (Throwable e) { |
| System.out.println("Unexpected exception:"); |
| e.printStackTrace(); |
| FAIL("Unexpected exception!"); |
| } |
| if (verbose) { System.out.println("Done!");} |
| } |
| |
| } // end class |