Victor Chang | 7322950 | 2020-09-17 13:39:19 +0100 | [diff] [blame] | 1 | // © 2016 and later: Unicode, Inc. and others. |
| 2 | // License & terms of use: http://www.unicode.org/copyright.html |
| 3 | /* |
| 4 | *************************************************************************** |
| 5 | * Copyright (C) 2008-2013, International Business Machines Corporation |
| 6 | * and others. All Rights Reserved. |
| 7 | *************************************************************************** |
| 8 | * |
| 9 | * uspoof_impl.h |
| 10 | * |
| 11 | * Implemenation header for spoof detection |
| 12 | * |
| 13 | */ |
| 14 | |
| 15 | #ifndef USPOOFIM_H |
| 16 | #define USPOOFIM_H |
| 17 | |
| 18 | #include "uassert.h" |
| 19 | #include "unicode/utypes.h" |
| 20 | #include "unicode/uspoof.h" |
| 21 | #include "unicode/uscript.h" |
| 22 | #include "unicode/udata.h" |
| 23 | #include "udataswp.h" |
| 24 | #include "utrie2.h" |
| 25 | |
| 26 | #if !UCONFIG_NO_NORMALIZATION |
| 27 | |
| 28 | #ifdef __cplusplus |
| 29 | |
| 30 | #include "capi_helper.h" |
| 31 | |
| 32 | U_NAMESPACE_BEGIN |
| 33 | |
| 34 | // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from |
| 35 | // a single input code point. This is function of the unicode.org data. |
| 36 | #define USPOOF_MAX_SKELETON_EXPANSION 20 |
| 37 | |
| 38 | // The default stack buffer size for copies or conversions or normalizations |
| 39 | // of input strings being checked. (Used in multiple places.) |
| 40 | #define USPOOF_STACK_BUFFER_SIZE 100 |
| 41 | |
| 42 | // Magic number for sanity checking spoof data. |
| 43 | #define USPOOF_MAGIC 0x3845fdef |
| 44 | |
| 45 | // Magic number for sanity checking spoof checkers. |
| 46 | #define USPOOF_CHECK_MAGIC 0x2734ecde |
| 47 | |
| 48 | class ScriptSet; |
| 49 | class SpoofData; |
| 50 | struct SpoofDataHeader; |
| 51 | class ConfusableDataUtils; |
| 52 | |
| 53 | /** |
| 54 | * Class SpoofImpl corresponds directly to the plain C API opaque type |
| 55 | * USpoofChecker. One can be cast to the other. |
| 56 | */ |
| 57 | class SpoofImpl : public UObject, |
| 58 | public IcuCApiHelper<USpoofChecker, SpoofImpl, USPOOF_MAGIC> { |
| 59 | public: |
| 60 | SpoofImpl(SpoofData *data, UErrorCode& status); |
| 61 | SpoofImpl(UErrorCode& status); |
| 62 | SpoofImpl(); |
| 63 | void construct(UErrorCode& status); |
| 64 | virtual ~SpoofImpl(); |
| 65 | |
| 66 | /** Copy constructor, used by the user level uspoof_clone() function. |
| 67 | */ |
| 68 | SpoofImpl(const SpoofImpl &src, UErrorCode &status); |
| 69 | |
| 70 | USpoofChecker *asUSpoofChecker(); |
| 71 | static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); |
| 72 | static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); |
| 73 | |
| 74 | /** Set and Get AllowedLocales, implementations of the corresponding API */ |
| 75 | void setAllowedLocales(const char *localesList, UErrorCode &status); |
| 76 | const char * getAllowedLocales(UErrorCode &status); |
| 77 | |
| 78 | // Add (union) to the UnicodeSet all of the characters for the scripts used for |
| 79 | // the specified locale. Part of the implementation of setAllowedLocales. |
| 80 | void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); |
| 81 | |
| 82 | // Functions implementing the features of UTS 39 section 5. |
| 83 | static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status); |
| 84 | void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const; |
| 85 | void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const; |
| 86 | void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const; |
| 87 | URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const; |
| 88 | |
| 89 | int32_t findHiddenOverlay(const UnicodeString& input, UErrorCode& status) const; |
| 90 | bool isIllegalCombiningDotLeadCharacter(UChar32 cp) const; |
| 91 | |
| 92 | /** parse a hex number. Untility used by the builders. */ |
| 93 | static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); |
| 94 | |
| 95 | static UClassID U_EXPORT2 getStaticClassID(void); |
| 96 | virtual UClassID getDynamicClassID(void) const; |
| 97 | |
| 98 | // |
| 99 | // Data Members |
| 100 | // |
| 101 | |
| 102 | int32_t fChecks; // Bit vector of checks to perform. |
| 103 | |
| 104 | SpoofData *fSpoofData; |
| 105 | |
| 106 | const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. |
| 107 | // for this Spoof Checker. Defaults to all chars. |
| 108 | |
| 109 | const char *fAllowedLocales; // The list of allowed locales. |
| 110 | URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. |
| 111 | }; |
| 112 | |
| 113 | /** |
| 114 | * Class CheckResult corresponds directly to the plain C API opaque type |
| 115 | * USpoofCheckResult. One can be cast to the other. |
| 116 | */ |
| 117 | class CheckResult : public UObject, |
| 118 | public IcuCApiHelper<USpoofCheckResult, CheckResult, USPOOF_CHECK_MAGIC> { |
| 119 | public: |
| 120 | CheckResult(); |
| 121 | virtual ~CheckResult(); |
| 122 | |
| 123 | USpoofCheckResult *asUSpoofCheckResult(); |
| 124 | static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status); |
| 125 | static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status); |
| 126 | |
| 127 | void clear(); |
| 128 | |
| 129 | // Used to convert this CheckResult to the older int32_t return value API |
| 130 | int32_t toCombinedBitmask(int32_t expectedChecks); |
| 131 | |
| 132 | // Data Members |
| 133 | int32_t fChecks; // Bit vector of checks that were failed. |
| 134 | UnicodeSet fNumerics; // Set of numerics found in the string. |
| 135 | URestrictionLevel fRestrictionLevel; // The restriction level of the string. |
| 136 | }; |
| 137 | |
| 138 | |
| 139 | // |
| 140 | // Confusable Mappings Data Structures, version 2.0 |
| 141 | // |
| 142 | // For the confusable data, we are essentially implementing a map, |
| 143 | // key: a code point |
| 144 | // value: a string. Most commonly one char in length, but can be more. |
| 145 | // |
| 146 | // The keys are stored as a sorted array of 32 bit ints. |
| 147 | // bits 0-23 a code point value |
| 148 | // bits 24-31 length of value string, in UChars (between 1 and 256 UChars). |
| 149 | // The key table is sorted in ascending code point order. (not on the |
| 150 | // 32 bit int value, the flag bits do not participate in the sorting.) |
| 151 | // |
| 152 | // Lookup is done by means of a binary search in the key table. |
| 153 | // |
| 154 | // The corresponding values are kept in a parallel array of 16 bit ints. |
| 155 | // If the value string is of length 1, it is literally in the value array. |
| 156 | // For longer strings, the value array contains an index into the strings table. |
| 157 | // |
| 158 | // String Table: |
| 159 | // The strings table contains all of the value strings (those of length two or greater) |
| 160 | // concatentated together into one long UChar (UTF-16) array. |
| 161 | // |
| 162 | // There is no nul character or other mark between adjacent strings. |
| 163 | // |
| 164 | //---------------------------------------------------------------------------- |
| 165 | // |
| 166 | // Changes from format version 1 to format version 2: |
| 167 | // 1) Removal of the whole-script confusable data tables. |
| 168 | // 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask. |
| 169 | // 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits. |
| 170 | // 4) Removal of the string lengths table since 8 bits is sufficient for the |
| 171 | // lengths of all entries in confusables.txt. |
| 172 | |
| 173 | |
| 174 | |
| 175 | // Internal functions for manipulating confusable data table keys |
| 176 | #define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58 |
| 177 | class ConfusableDataUtils { |
| 178 | public: |
| 179 | inline static UChar32 keyToCodePoint(int32_t key) { |
| 180 | return key & 0x00ffffff; |
| 181 | } |
| 182 | inline static int32_t keyToLength(int32_t key) { |
| 183 | return ((key & 0xff000000) >> 24) + 1; |
| 184 | } |
| 185 | inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) { |
| 186 | U_ASSERT((codePoint & 0x00ffffff) == codePoint); |
| 187 | U_ASSERT(length <= 256); |
| 188 | return codePoint | ((length - 1) << 24); |
| 189 | } |
| 190 | }; |
| 191 | |
| 192 | |
| 193 | //------------------------------------------------------------------------------------- |
| 194 | // |
| 195 | // SpoofData |
| 196 | // |
| 197 | // A small class that wraps the raw (usually memory mapped) spoof data. |
| 198 | // Serves two primary functions: |
| 199 | // 1. Convenience. Contains real pointers to the data, to avoid dealing with |
| 200 | // the offsets in the raw data. |
| 201 | // 2. Reference counting. When a spoof checker is cloned, the raw data is shared |
| 202 | // and must be retained until all checkers using the data are closed. |
| 203 | // Nothing in this struct includes state that is specific to any particular |
| 204 | // USpoofDetector object. |
| 205 | // |
| 206 | //--------------------------------------------------------------------------------------- |
| 207 | class SpoofData: public UMemory { |
| 208 | public: |
| 209 | static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data. |
| 210 | static void releaseDefault(); // Cleanup reference to default spoof data. |
| 211 | |
| 212 | SpoofData(UErrorCode &status); // Create new spoof data wrapper. |
| 213 | // Only used when building new data from rules. |
| 214 | |
| 215 | // Constructor for use when creating from prebuilt default data. |
| 216 | // A UDataMemory is what the ICU internal data loading functions provide. |
| 217 | // The udm is adopted by the SpoofData. |
| 218 | SpoofData(UDataMemory *udm, UErrorCode &status); |
| 219 | |
| 220 | // Constructor for use when creating from serialized data. |
| 221 | // |
| 222 | SpoofData(const void *serializedData, int32_t length, UErrorCode &status); |
| 223 | |
| 224 | // Check raw Spoof Data Version compatibility. |
Victor Chang | ce4bf3c | 2021-01-19 16:34:24 +0000 | [diff] [blame] | 225 | // Return true it looks good. |
Victor Chang | 7322950 | 2020-09-17 13:39:19 +0100 | [diff] [blame] | 226 | UBool validateDataVersion(UErrorCode &status) const; |
| 227 | |
| 228 | ~SpoofData(); // Destructor not normally used. |
| 229 | // Use removeReference() instead. |
| 230 | // Reference Counting functions. |
| 231 | // Clone of a user-level spoof detector increments the ref count on the data. |
| 232 | // Close of a user-level spoof detector decrements the ref count. |
| 233 | // If the data is owned by us, it will be deleted when count goes to zero. |
| 234 | SpoofData *addReference(); |
| 235 | void removeReference(); |
| 236 | |
| 237 | // Reset all fields to an initial state. |
| 238 | // Called from the top of all constructors. |
| 239 | void reset(); |
| 240 | |
| 241 | // Copy this instance's raw data buffer to the specified address. |
| 242 | int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const; |
| 243 | |
| 244 | // Get the total number of bytes of data backed by this SpoofData. |
| 245 | // Not to be confused with length, which returns the number of confusable entries. |
| 246 | int32_t size() const; |
| 247 | |
| 248 | // Get the confusable skeleton transform for a single code point. |
| 249 | // The result is a string with a length between 1 and 18 as of Unicode 9. |
| 250 | // This is the main public endpoint for this class. |
| 251 | // @return The length in UTF-16 code units of the substition string. |
| 252 | int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const; |
| 253 | |
| 254 | // Get the number of confusable entries in this SpoofData. |
| 255 | int32_t length() const; |
| 256 | |
| 257 | // Get the code point (key) at the specified index. |
| 258 | UChar32 codePointAt(int32_t index) const; |
| 259 | |
| 260 | // Get the confusable skeleton (value) at the specified index. |
| 261 | // Append it to the specified UnicodeString&. |
| 262 | // @return The length in UTF-16 code units of the skeleton string. |
| 263 | int32_t appendValueTo(int32_t index, UnicodeString& dest) const; |
| 264 | |
| 265 | private: |
| 266 | // Reserve space in the raw data. For use by builder when putting together a |
| 267 | // new set of data. Init the new storage to zero, to prevent inconsistent |
| 268 | // results if it is not all otherwise set by the requester. |
| 269 | // Return: |
| 270 | // pointer to the new space that was added by this function. |
| 271 | void *reserveSpace(int32_t numBytes, UErrorCode &status); |
| 272 | |
| 273 | // initialize the pointers from this object to the raw data. |
| 274 | void initPtrs(UErrorCode &status); |
| 275 | |
| 276 | SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data |
| 277 | UBool fDataOwned; // True if the raw data is owned, and needs |
| 278 | // to be deleted when refcount goes to zero. |
| 279 | UDataMemory *fUDM; // If not NULL, our data came from a |
| 280 | // UDataMemory, which we must close when |
| 281 | // we are done. |
| 282 | |
| 283 | uint32_t fMemLimit; // Limit of available raw data space |
| 284 | u_atomic_int32_t fRefCount; |
| 285 | |
| 286 | // Confusable data |
| 287 | int32_t *fCFUKeys; |
| 288 | uint16_t *fCFUValues; |
| 289 | UChar *fCFUStrings; |
| 290 | |
| 291 | friend class ConfusabledataBuilder; |
| 292 | }; |
| 293 | |
| 294 | //--------------------------------------------------------------------------------------- |
| 295 | // |
| 296 | // Raw Binary Data Formats, as loaded from the ICU data file, |
| 297 | // or as built by the builder. |
| 298 | // |
| 299 | //--------------------------------------------------------------------------------------- |
| 300 | struct SpoofDataHeader { |
| 301 | int32_t fMagic; // (0x3845fdef) |
| 302 | uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo |
| 303 | // if there is one associated with this data. |
| 304 | int32_t fLength; // Total lenght in bytes of this spoof data, |
| 305 | // including all sections, not just the header. |
| 306 | |
| 307 | // The following four sections refer to data representing the confusable data |
| 308 | // from the Unicode.org data from "confusables.txt" |
| 309 | |
| 310 | int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) |
| 311 | int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) |
| 312 | |
| 313 | // TODO: change name to fCFUValues, for consistency. |
| 314 | int32_t fCFUStringIndex; // byte offset to String Indexes table |
| 315 | int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) |
| 316 | // (number of entries must be same as in Keys table |
| 317 | |
| 318 | int32_t fCFUStringTable; // byte offset of String table |
| 319 | int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) |
| 320 | |
| 321 | // The following sections are for data from xidmodifications.txt |
| 322 | |
| 323 | int32_t unused[15]; // Padding, Room for Expansion |
| 324 | }; |
| 325 | |
| 326 | |
| 327 | |
| 328 | U_NAMESPACE_END |
| 329 | #endif /* __cplusplus */ |
| 330 | |
| 331 | /** |
| 332 | * Endianness swap function for binary spoof data. |
| 333 | * @internal |
| 334 | */ |
| 335 | U_CAPI int32_t U_EXPORT2 |
| 336 | uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, |
| 337 | UErrorCode *status); |
| 338 | |
| 339 | |
| 340 | #endif |
| 341 | |
| 342 | #endif /* USPOOFIM_H */ |
| 343 | |