Victor Chang | 7322950 | 2020-09-17 13:39:19 +0100 | [diff] [blame] | 1 | // © 2016 and later: Unicode, Inc. and others. |
| 2 | // License & terms of use: http://www.unicode.org/copyright.html |
| 3 | /* |
| 4 | ******************************************************************************* |
| 5 | * Copyright (C) 2010-2015, International Business Machines |
| 6 | * Corporation and others. All Rights Reserved. |
| 7 | ******************************************************************************* |
| 8 | * collationdata.h |
| 9 | * |
| 10 | * created on: 2010oct27 |
| 11 | * created by: Markus W. Scherer |
| 12 | */ |
| 13 | |
| 14 | #ifndef __COLLATIONDATA_H__ |
| 15 | #define __COLLATIONDATA_H__ |
| 16 | |
| 17 | #include "unicode/utypes.h" |
| 18 | |
| 19 | #if !UCONFIG_NO_COLLATION |
| 20 | |
| 21 | #include "unicode/ucol.h" |
| 22 | #include "unicode/uniset.h" |
| 23 | #include "collation.h" |
| 24 | #include "normalizer2impl.h" |
| 25 | #include "utrie2.h" |
| 26 | |
| 27 | struct UDataMemory; |
| 28 | |
| 29 | U_NAMESPACE_BEGIN |
| 30 | |
| 31 | class UVector32; |
| 32 | |
| 33 | /** |
| 34 | * Collation data container. |
| 35 | * Immutable data created by a CollationDataBuilder, or loaded from a file, |
| 36 | * or deserialized from API-provided binary data. |
| 37 | * |
| 38 | * Includes data for the collation base (root/default), aliased if this is not the base. |
| 39 | */ |
| 40 | struct U_I18N_API CollationData : public UMemory { |
| 41 | // Note: The ucadata.icu loader could discover the reserved ranges by setting an array |
| 42 | // parallel with the ranges, and resetting ranges that are indexed. |
| 43 | // The reordering builder code could clone the resulting template array. |
| 44 | enum { |
| 45 | REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14, |
| 46 | REORDER_RESERVED_AFTER_LATIN |
| 47 | }; |
| 48 | |
| 49 | enum { |
| 50 | MAX_NUM_SPECIAL_REORDER_CODES = 8, |
| 51 | /** C++ only, data reader check scriptStartsLength. */ |
| 52 | MAX_NUM_SCRIPT_RANGES = 256 |
| 53 | }; |
| 54 | |
| 55 | CollationData(const Normalizer2Impl &nfc) |
| 56 | : trie(NULL), |
| 57 | ce32s(NULL), ces(NULL), contexts(NULL), base(NULL), |
| 58 | jamoCE32s(NULL), |
| 59 | nfcImpl(nfc), |
| 60 | numericPrimary(0x12000000), |
| 61 | ce32sLength(0), cesLength(0), contextsLength(0), |
| 62 | compressibleBytes(NULL), |
| 63 | unsafeBackwardSet(NULL), |
| 64 | fastLatinTable(NULL), fastLatinTableLength(0), |
| 65 | numScripts(0), scriptsIndex(NULL), scriptStarts(NULL), scriptStartsLength(0), |
| 66 | rootElements(NULL), rootElementsLength(0) {} |
| 67 | |
| 68 | uint32_t getCE32(UChar32 c) const { |
| 69 | return UTRIE2_GET32(trie, c); |
| 70 | } |
| 71 | |
| 72 | uint32_t getCE32FromSupplementary(UChar32 c) const { |
| 73 | return UTRIE2_GET32_FROM_SUPP(trie, c); |
| 74 | } |
| 75 | |
| 76 | UBool isDigit(UChar32 c) const { |
| 77 | return c < 0x660 ? c <= 0x39 && 0x30 <= c : |
| 78 | Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG); |
| 79 | } |
| 80 | |
| 81 | UBool isUnsafeBackward(UChar32 c, UBool numeric) const { |
| 82 | return unsafeBackwardSet->contains(c) || (numeric && isDigit(c)); |
| 83 | } |
| 84 | |
| 85 | UBool isCompressibleLeadByte(uint32_t b) const { |
| 86 | return compressibleBytes[b]; |
| 87 | } |
| 88 | |
| 89 | inline UBool isCompressiblePrimary(uint32_t p) const { |
| 90 | return isCompressibleLeadByte(p >> 24); |
| 91 | } |
| 92 | |
| 93 | /** |
| 94 | * Returns the CE32 from two contexts words. |
| 95 | * Access to the defaultCE32 for contraction and prefix matching. |
| 96 | */ |
| 97 | static uint32_t readCE32(const UChar *p) { |
| 98 | return ((uint32_t)p[0] << 16) | p[1]; |
| 99 | } |
| 100 | |
| 101 | /** |
| 102 | * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG). |
| 103 | * Requires that ce32 is special. |
| 104 | */ |
| 105 | uint32_t getIndirectCE32(uint32_t ce32) const; |
| 106 | /** |
| 107 | * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG), |
| 108 | * if ce32 is special. |
| 109 | */ |
| 110 | uint32_t getFinalCE32(uint32_t ce32) const; |
| 111 | |
| 112 | /** |
| 113 | * Computes a CE from c's ce32 which has the OFFSET_TAG. |
| 114 | */ |
| 115 | int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const { |
| 116 | int64_t dataCE = ces[Collation::indexFromCE32(ce32)]; |
| 117 | return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE)); |
| 118 | } |
| 119 | |
| 120 | /** |
| 121 | * Returns the single CE that c maps to. |
| 122 | * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE. |
| 123 | */ |
| 124 | int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const; |
| 125 | |
| 126 | /** |
| 127 | * Returns the FCD16 value for code point c. c must be >= 0. |
| 128 | */ |
| 129 | uint16_t getFCD16(UChar32 c) const { |
| 130 | return nfcImpl.getFCD16(c); |
| 131 | } |
| 132 | |
| 133 | /** |
| 134 | * Returns the first primary for the script's reordering group. |
| 135 | * @return the primary with only the first primary lead byte of the group |
| 136 | * (not necessarily an actual root collator primary weight), |
| 137 | * or 0 if the script is unknown |
| 138 | */ |
| 139 | uint32_t getFirstPrimaryForGroup(int32_t script) const; |
| 140 | |
| 141 | /** |
| 142 | * Returns the last primary for the script's reordering group. |
| 143 | * @return the last primary of the group |
| 144 | * (not an actual root collator primary weight), |
| 145 | * or 0 if the script is unknown |
| 146 | */ |
| 147 | uint32_t getLastPrimaryForGroup(int32_t script) const; |
| 148 | |
| 149 | /** |
| 150 | * Finds the reordering group which contains the primary weight. |
| 151 | * @return the first script of the group, or -1 if the weight is beyond the last group |
| 152 | */ |
| 153 | int32_t getGroupForPrimary(uint32_t p) const; |
| 154 | |
| 155 | int32_t getEquivalentScripts(int32_t script, |
| 156 | int32_t dest[], int32_t capacity, UErrorCode &errorCode) const; |
| 157 | |
| 158 | /** |
| 159 | * Writes the permutation of primary-weight ranges |
| 160 | * for the given reordering of scripts and groups. |
| 161 | * The caller checks for illegal arguments and |
| 162 | * takes care of [DEFAULT] and memory allocation. |
| 163 | * |
| 164 | * Each list element will be a (limit, offset) pair as described |
| 165 | * for the CollationSettings::reorderRanges. |
| 166 | * The list will be empty if no ranges are reordered. |
| 167 | */ |
| 168 | void makeReorderRanges(const int32_t *reorder, int32_t length, |
| 169 | UVector32 &ranges, UErrorCode &errorCode) const; |
| 170 | |
| 171 | /** @see jamoCE32s */ |
| 172 | static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27; |
| 173 | |
| 174 | /** Main lookup trie. */ |
| 175 | const UTrie2 *trie; |
| 176 | /** |
| 177 | * Array of CE32 values. |
| 178 | * At index 0 there must be CE32(U+0000) |
| 179 | * to support U+0000's special-tag for NUL-termination handling. |
| 180 | */ |
| 181 | const uint32_t *ce32s; |
| 182 | /** Array of CE values for expansions and OFFSET_TAG. */ |
| 183 | const int64_t *ces; |
| 184 | /** Array of prefix and contraction-suffix matching data. */ |
| 185 | const UChar *contexts; |
| 186 | /** Base collation data, or NULL if this data itself is a base. */ |
| 187 | const CollationData *base; |
| 188 | /** |
| 189 | * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T. |
| 190 | * They are normally simple CE32s, rarely expansions. |
| 191 | * For fast handling of HANGUL_TAG. |
| 192 | */ |
| 193 | const uint32_t *jamoCE32s; |
| 194 | const Normalizer2Impl &nfcImpl; |
| 195 | /** The single-byte primary weight (xx000000) for numeric collation. */ |
| 196 | uint32_t numericPrimary; |
| 197 | |
| 198 | int32_t ce32sLength; |
| 199 | int32_t cesLength; |
| 200 | int32_t contextsLength; |
| 201 | |
| 202 | /** 256 flags for which primary-weight lead bytes are compressible. */ |
| 203 | const UBool *compressibleBytes; |
| 204 | /** |
| 205 | * Set of code points that are unsafe for starting string comparison after an identical prefix, |
| 206 | * or in backwards CE iteration. |
| 207 | */ |
| 208 | const UnicodeSet *unsafeBackwardSet; |
| 209 | |
| 210 | /** |
| 211 | * Fast Latin table for common-Latin-text string comparisons. |
| 212 | * Data structure see class CollationFastLatin. |
| 213 | */ |
| 214 | const uint16_t *fastLatinTable; |
| 215 | int32_t fastLatinTableLength; |
| 216 | |
| 217 | /** |
| 218 | * Data for scripts and reordering groups. |
| 219 | * Uses include building a reordering permutation table and |
| 220 | * providing script boundaries to AlphabeticIndex. |
| 221 | */ |
| 222 | int32_t numScripts; |
| 223 | /** |
| 224 | * The length of scriptsIndex is numScripts+16. |
| 225 | * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts. |
| 226 | * 16 special reorder codes (not all used) are mapped starting at numScripts. |
| 227 | * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit. |
| 228 | * There are special codes at the end for reorder-reserved primary ranges. |
| 229 | * |
| 230 | * Multiple scripts may share a range and index, for example Hira & Kana. |
| 231 | */ |
| 232 | const uint16_t *scriptsIndex; |
| 233 | /** |
| 234 | * Start primary weight (top 16 bits only) for a group/script/reserved range |
| 235 | * indexed by scriptsIndex. |
| 236 | * The first range (separators & terminators) and the last range (trailing weights) |
| 237 | * are not reorderable, and no scriptsIndex entry points to them. |
| 238 | */ |
| 239 | const uint16_t *scriptStarts; |
| 240 | int32_t scriptStartsLength; |
| 241 | |
| 242 | /** |
| 243 | * Collation elements in the root collator. |
| 244 | * Used by the CollationRootElements class. The data structure is described there. |
| 245 | * NULL in a tailoring. |
| 246 | */ |
| 247 | const uint32_t *rootElements; |
| 248 | int32_t rootElementsLength; |
| 249 | |
| 250 | private: |
| 251 | int32_t getScriptIndex(int32_t script) const; |
| 252 | void makeReorderRanges(const int32_t *reorder, int32_t length, |
| 253 | UBool latinMustMove, |
| 254 | UVector32 &ranges, UErrorCode &errorCode) const; |
| 255 | int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const; |
| 256 | int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const; |
| 257 | }; |
| 258 | |
| 259 | U_NAMESPACE_END |
| 260 | |
| 261 | #endif // !UCONFIG_NO_COLLATION |
| 262 | #endif // __COLLATIONDATA_H__ |