Victor Chang | 7322950 | 2020-09-17 13:39:19 +0100 | [diff] [blame] | 1 | // © 2016 and later: Unicode, Inc. and others. |
| 2 | // License & terms of use: http://www.unicode.org/copyright.html |
| 3 | /* |
| 4 | ****************************************************************************** |
| 5 | * |
| 6 | * Copyright (C) 2000-2013, International Business Machines |
| 7 | * Corporation and others. All Rights Reserved. |
| 8 | * |
| 9 | ****************************************************************************** |
| 10 | * file name: ucnvmbcs.h |
| 11 | * encoding: UTF-8 |
| 12 | * tab size: 8 (not used) |
| 13 | * indentation:4 |
| 14 | * |
| 15 | * created on: 2000jul07 |
| 16 | * created by: Markus W. Scherer |
| 17 | */ |
| 18 | |
| 19 | #ifndef __UCNVMBCS_H__ |
| 20 | #define __UCNVMBCS_H__ |
| 21 | |
| 22 | #include "unicode/utypes.h" |
| 23 | |
| 24 | #if !UCONFIG_NO_CONVERSION |
| 25 | |
| 26 | #include "unicode/ucnv.h" |
| 27 | #include "ucnv_cnv.h" |
| 28 | #include "ucnv_ext.h" |
| 29 | |
| 30 | /** |
| 31 | * ICU conversion (.cnv) data file structure, following the usual UDataInfo |
| 32 | * header. |
| 33 | * |
| 34 | * Format version: 6.2 |
| 35 | * |
| 36 | * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, |
| 37 | * min/max bytes per character, etc. |
| 38 | * see ucnv_bld.h |
| 39 | * |
| 40 | * -------------------- |
| 41 | * |
| 42 | * The static data is followed by conversionType-specific data structures. |
| 43 | * At the moment, there are only variations of MBCS converters. They all have |
| 44 | * the same toUnicode structures, while the fromUnicode structures for SBCS |
| 45 | * differ from those for other MBCS-style converters. |
| 46 | * |
| 47 | * _MBCSHeader.version 5 is optional and not backward-compatible |
| 48 | * (as usual for changes in the major version field). |
| 49 | * |
| 50 | * Versions 5.m work like versions 4.m except: |
| 51 | * - The _MBCSHeader has variable length (and is always longer than in version 4). |
| 52 | * See the struct _MBCSHeader further description below. |
| 53 | * - There is a set of flags which indicate further incompatible changes. |
| 54 | * (Reader code must reject the file if it does not recognize them all.) |
| 55 | * - In particular, one of these flags indicates that most of the fromUnicode |
| 56 | * data is missing and must be reconstituted from the toUnicode data |
| 57 | * and from the utf8Friendly mbcsIndex at load time. |
| 58 | * (This only works with a utf8Friendly table.) |
| 59 | * In this case, makeconv may increase maxFastUChar automatically to U+FFFF. |
| 60 | * |
| 61 | * The first of these versions is 5.3, which is like 4.3 except for the differences above. |
| 62 | * |
| 63 | * When possible, makeconv continues to generate version 4.m files. |
| 64 | * |
| 65 | * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4) |
| 66 | * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details. |
| 67 | * |
| 68 | * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures |
| 69 | * slightly and optionally adds a table for conversion to MBCS (non-SBCS) |
| 70 | * charsets. |
| 71 | * |
| 72 | * The modifications are to make the data utf8Friendly. Not every 4.3 file |
| 73 | * file contains utf8Friendly data. |
| 74 | * It is utf8Friendly if _MBCSHeader.version[2]!=0. |
| 75 | * In this case, the data structures are utf8Friendly up to the code point |
| 76 | * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff) |
| 77 | * |
| 78 | * A utf8Friendly file has fromUnicode stage 3 entries for code points up to |
| 79 | * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from |
| 80 | * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries. |
| 81 | * |
| 82 | * In addition, a utf8Friendly MBCS file contains an additional |
| 83 | * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; |
| 84 | * which replaces the stage 1 and 2 tables for indexing with bits from the |
| 85 | * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table, |
| 86 | * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks |
| 87 | * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to |
| 88 | * the extension data structure. This also allows for faster roundtrip |
| 89 | * conversion from UTF-16. |
| 90 | * |
| 91 | * SBCS files do not contain an additional sbcsIndex[] array because the |
| 92 | * proportional size increase would be noticeable, but the runtime |
| 93 | * code builds one for the code point range for which the runtime conversion |
| 94 | * code is optimized. |
| 95 | * |
| 96 | * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv |
| 97 | * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in |
| 98 | * the file, a larger maxFastUChar only affects stage 3 block allocation size |
| 99 | * and is free in empty blocks. (Larger blocks with sparse contents cause larger |
| 100 | * files.) U+1FFF includes almost all of the small scripts. |
| 101 | * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with |
| 102 | * 0xe0. This includes most scripts with legacy SBCS charsets. |
| 103 | * The initial runtime implementation using 4.3 files only builds an sbcsIndex |
| 104 | * for code points up to U+0FFF. |
| 105 | * |
| 106 | * For MBCS, maxFastUChar should be at least U+D7FF (=initial value). |
| 107 | * This boundary is convenient because practically all of the commonly used |
| 108 | * characters are below it, and because it is the boundary to surrogate |
| 109 | * code points, above which special handling is necessary anyway. |
| 110 | * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.) |
| 111 | * |
| 112 | * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be |
| 113 | * useful especially for conversion from UTF-8 when the input can be assumed |
| 114 | * to be valid, because the surrogate range would then not have to be |
| 115 | * checked. |
| 116 | * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value |
| 117 | * overflow because with the all-unassigned block 0 and nearly full mappings |
| 118 | * from the BMP it is theoretically possible that an index into stage 3 |
| 119 | * exceeds 16 bits.) |
| 120 | * |
| 121 | * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. |
| 122 | * If it is present, then an ICU version reading header versions 4.0 or 4.1 |
| 123 | * will be able to use the base table and ignore the extension. |
| 124 | * |
| 125 | * The unicodeMask in the static data is part of the base table data structure. |
| 126 | * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the |
| 127 | * fromUnicode stage 1 array. |
| 128 | * The static data unicodeMask refers only to the base table's properties if |
| 129 | * a base table is included. |
| 130 | * In an extension-only file, the static data unicodeMask is 0. |
| 131 | * The extension data indexes have a separate field with the unicodeMask flags. |
| 132 | * |
| 133 | * MBCS-style data structure following the static data. |
| 134 | * Offsets are counted in bytes from the beginning of the MBCS header structure. |
| 135 | * Details about usage in comments in ucnvmbcs.c. |
| 136 | * |
| 137 | * struct _MBCSHeader (see the definition in this header file below) |
| 138 | * contains 32-bit fields as follows: |
| 139 | * 8 values: |
| 140 | * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0) |
| 141 | * 1 uint32_t countStates |
| 142 | * 2 uint32_t countToUFallbacks |
| 143 | * 3 uint32_t offsetToUCodeUnits |
| 144 | * 4 uint32_t offsetFromUTable |
| 145 | * 5 uint32_t offsetFromUBytes |
| 146 | * 6 uint32_t flags, bits: |
| 147 | * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher |
| 148 | * 0 for older versions and if |
| 149 | * there is not extension structure |
| 150 | * 7.. 0 outputType |
| 151 | * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher |
| 152 | * counts bytes in fromUBytes[] |
| 153 | * |
| 154 | * New and required in version 5: |
| 155 | * 8 uint32_t options, bits: |
| 156 | * 31..16 reserved for flags that can be added without breaking |
| 157 | * backward compatibility |
| 158 | * 15.. 6 reserved for flags whose addition will break |
| 159 | * backward compatibility |
| 160 | * 6 MBCS_OPT_FROM_U -- if set, |
| 161 | * then most of the fromUnicode data is omitted; |
| 162 | * fullStage2Length is present and the missing |
| 163 | * bottom part of stage 2 must be reconstituted from |
| 164 | * the toUnicode data; |
| 165 | * stage 3 is missing completely as well; |
| 166 | * not used for SBCS tables |
| 167 | * 5.. 0 length of the _MBCSHeader (number of uint32_t) |
| 168 | * |
| 169 | * New and optional in version 5: |
| 170 | * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set |
| 171 | * specifies the full length of stage 2 |
| 172 | * including the omitted part |
| 173 | * |
| 174 | * if(outputType==MBCS_OUTPUT_EXT_ONLY) { |
| 175 | * -- base table name for extension-only table |
| 176 | * char baseTableName[variable]; -- with NUL plus padding for 4-alignment |
| 177 | * |
| 178 | * -- all _MBCSHeader fields except for version and flags are 0 |
| 179 | * } else { |
| 180 | * -- normal base table with optional extension |
| 181 | * |
| 182 | * int32_t stateTable[countStates][256]; |
| 183 | * |
| 184 | * struct _MBCSToUFallback { (fallbacks are sorted by offset) |
| 185 | * uint32_t offset; |
| 186 | * UChar32 codePoint; |
| 187 | * } toUFallbacks[countToUFallbacks]; |
| 188 | * |
| 189 | * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; |
| 190 | * (padded to an even number of units) |
| 191 | * |
| 192 | * -- stage 1 tables |
| 193 | * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { |
| 194 | * -- stage 1 table for all of Unicode |
| 195 | * uint16_t fromUTable[0x440]; (32-bit-aligned) |
| 196 | * } else { |
| 197 | * -- BMP-only tables have a smaller stage 1 table |
| 198 | * uint16_t fromUTable[0x40]; (32-bit-aligned) |
| 199 | * } |
| 200 | * |
| 201 | * -- stage 2 tables |
| 202 | * length determined by top of stage 1 and bottom of stage 3 tables |
| 203 | * if(outputType==MBCS_OUTPUT_1) { |
| 204 | * -- SBCS: pure indexes |
| 205 | * uint16_t stage 2 indexes[?]; |
| 206 | * } else { |
| 207 | * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes |
| 208 | * uint32_t stage 2 flags and indexes[?]; |
| 209 | * if(options&MBCS_OPT_NO_FROM_U) { |
| 210 | * stage 2 really has length fullStage2Length |
| 211 | * and the omitted lower part must be reconstituted from |
| 212 | * the toUnicode data |
| 213 | * } |
| 214 | * } |
| 215 | * |
| 216 | * -- stage 3 tables with byte results |
| 217 | * if(outputType==MBCS_OUTPUT_1) { |
| 218 | * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c |
| 219 | * uint16_t fromUBytes[fromUBytesLength/2]; |
| 220 | * } else if(!(options&MBCS_OPT_NO_FROM_U)) { |
| 221 | * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c |
| 222 | * uint8_t fromUBytes[fromUBytesLength]; or |
| 223 | * uint16_t fromUBytes[fromUBytesLength/2]; or |
| 224 | * uint32_t fromUBytes[fromUBytesLength/4]; |
| 225 | * } else { |
| 226 | * fromUBytes[] must be reconstituted from the toUnicode data |
| 227 | * } |
| 228 | * |
| 229 | * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher |
| 230 | * if(outputType!=MBCS_OUTPUT_1 && |
| 231 | * _MBCSHeader.version[1]>=3 && |
| 232 | * (maxFastUChar=_MBCSHeader.version[2])!=0 |
| 233 | * ) { |
| 234 | * maxFastUChar=(maxFastUChar<<8)|0xff; |
| 235 | * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; |
| 236 | * } |
| 237 | * } |
| 238 | * |
| 239 | * -- extension table, details see ucnv_ext.h |
| 240 | * int32_t indexes[>=32]; ... |
| 241 | */ |
| 242 | |
| 243 | /* MBCS converter data and state -------------------------------------------- */ |
| 244 | |
| 245 | enum { |
| 246 | MBCS_MAX_STATE_COUNT=128 |
| 247 | }; |
| 248 | |
| 249 | /** |
| 250 | * MBCS action codes for conversions to Unicode. |
| 251 | * These values are in bits 23..20 of the state table entries. |
| 252 | */ |
| 253 | enum { |
| 254 | MBCS_STATE_VALID_DIRECT_16, |
| 255 | MBCS_STATE_VALID_DIRECT_20, |
| 256 | |
| 257 | MBCS_STATE_FALLBACK_DIRECT_16, |
| 258 | MBCS_STATE_FALLBACK_DIRECT_20, |
| 259 | |
| 260 | MBCS_STATE_VALID_16, |
| 261 | MBCS_STATE_VALID_16_PAIR, |
| 262 | |
| 263 | MBCS_STATE_UNASSIGNED, |
| 264 | MBCS_STATE_ILLEGAL, |
| 265 | |
| 266 | MBCS_STATE_CHANGE_ONLY |
| 267 | }; |
| 268 | |
| 269 | /* Macros for state table entries */ |
| 270 | #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset)) |
| 271 | #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset)) |
| 272 | #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset)) |
| 273 | |
| 274 | #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value)) |
| 275 | #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000) |
| 276 | #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L)) |
| 277 | #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value)) |
| 278 | #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value)) |
| 279 | |
| 280 | #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) |
| 281 | |
| 282 | #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) |
| 283 | |
| 284 | #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) |
| 285 | #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) |
| 286 | |
| 287 | #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24) |
| 288 | #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) |
| 289 | |
| 290 | #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) |
| 291 | #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) |
| 292 | #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf) |
| 293 | #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) |
| 294 | #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) |
| 295 | |
| 296 | #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0) |
| 297 | |
| 298 | /* single-byte fromUnicode: get the 16-bit result word */ |
| 299 | #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] |
| 300 | |
| 301 | /* single-byte fromUnicode using the sbcsIndex */ |
| 302 | #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] |
| 303 | |
| 304 | /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */ |
| 305 | #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ] |
| 306 | |
| 307 | /* multi-byte fromUnicode: get the 32-bit stage 2 entry */ |
| 308 | #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] |
| 309 | #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) |
| 310 | |
| 311 | #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] |
| 312 | #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] |
| 313 | |
| 314 | #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) |
| 315 | |
| 316 | /* double-byte fromUnicode using the mbcsIndex */ |
| 317 | #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] |
| 318 | |
| 319 | /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */ |
| 320 | #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ] |
| 321 | |
| 322 | |
| 323 | /** |
| 324 | * MBCS output types for conversions from Unicode. |
| 325 | * These per-converter types determine the storage method in stage 3 of the lookup table, |
| 326 | * mostly how many bytes are stored per entry. |
| 327 | */ |
| 328 | enum { |
| 329 | MBCS_OUTPUT_1, /* 0 */ |
| 330 | MBCS_OUTPUT_2, /* 1 */ |
| 331 | MBCS_OUTPUT_3, /* 2 */ |
| 332 | MBCS_OUTPUT_4, /* 3 */ |
| 333 | |
| 334 | MBCS_OUTPUT_3_EUC=8, /* 8 */ |
| 335 | MBCS_OUTPUT_4_EUC, /* 9 */ |
| 336 | |
| 337 | MBCS_OUTPUT_2_SISO=12, /* c */ |
| 338 | MBCS_OUTPUT_2_HZ, /* d */ |
| 339 | |
| 340 | MBCS_OUTPUT_EXT_ONLY, /* e */ |
| 341 | |
| 342 | MBCS_OUTPUT_COUNT, |
| 343 | |
| 344 | MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */ |
| 345 | }; |
| 346 | |
| 347 | /** |
| 348 | * Fallbacks to Unicode are stored outside the normal state table and code point structures |
| 349 | * in a vector of items of this type. They are sorted by offset. |
| 350 | */ |
| 351 | typedef struct { |
| 352 | uint32_t offset; |
| 353 | UChar32 codePoint; |
| 354 | } _MBCSToUFallback; |
| 355 | |
| 356 | /** Constants for fast and UTF-8-friendly conversion. */ |
| 357 | enum { |
| 358 | SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */ |
| 359 | SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */ |
| 360 | MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */ |
| 361 | MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */ |
| 362 | }; |
| 363 | |
| 364 | /** |
| 365 | * This is the MBCS part of the UConverterTable union (a runtime data structure). |
| 366 | * It keeps all the per-converter data and points into the loaded mapping tables. |
| 367 | * |
| 368 | * utf8Friendly data structures added with _MBCSHeader.version 4.3 |
| 369 | */ |
| 370 | typedef struct UConverterMBCSTable { |
| 371 | /* toUnicode */ |
| 372 | uint8_t countStates, dbcsOnlyState, stateTableOwned; |
| 373 | uint32_t countToUFallbacks; |
| 374 | |
| 375 | const int32_t (*stateTable)/*[countStates]*/[256]; |
| 376 | int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */ |
| 377 | const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/; |
| 378 | const _MBCSToUFallback *toUFallbacks; |
| 379 | |
| 380 | /* fromUnicode */ |
| 381 | const uint16_t *fromUnicodeTable; |
| 382 | const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */ |
| 383 | uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */ |
| 384 | const uint8_t *fromUnicodeBytes; |
| 385 | uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ |
| 386 | uint32_t fromUBytesLength; |
| 387 | uint8_t outputType, unicodeMask; |
| 388 | UBool utf8Friendly; /* for utf8Friendly data */ |
| 389 | UChar maxFastUChar; /* for utf8Friendly data */ |
| 390 | |
| 391 | /* roundtrips */ |
| 392 | uint32_t asciiRoundtrips; |
| 393 | |
| 394 | /* reconstituted data that was omitted from the .cnv file */ |
| 395 | uint8_t *reconstitutedData; |
| 396 | |
| 397 | /* converter name for swaplfnl */ |
| 398 | char *swapLFNLName; |
| 399 | |
| 400 | /* extension data */ |
| 401 | struct UConverterSharedData *baseSharedData; |
| 402 | const int32_t *extIndexes; |
| 403 | } UConverterMBCSTable; |
| 404 | |
| 405 | #define UCNV_MBCS_TABLE_INITIALIZER { \ |
| 406 | /* toUnicode */ \ |
| 407 | 0, 0, 0, \ |
| 408 | 0, \ |
| 409 | \ |
| 410 | NULL, \ |
| 411 | NULL, \ |
| 412 | NULL, \ |
| 413 | NULL, \ |
| 414 | \ |
| 415 | /* fromUnicode */ \ |
| 416 | NULL, \ |
| 417 | NULL, \ |
| 418 | { 0 }, \ |
| 419 | NULL, \ |
| 420 | NULL, \ |
| 421 | 0, \ |
| 422 | 0, 0, \ |
Victor Chang | ce4bf3c | 2021-01-19 16:34:24 +0000 | [diff] [blame] | 423 | false, \ |
Victor Chang | 7322950 | 2020-09-17 13:39:19 +0100 | [diff] [blame] | 424 | 0, \ |
| 425 | \ |
| 426 | /* roundtrips */ \ |
| 427 | 0, \ |
| 428 | \ |
| 429 | /* reconstituted data that was omitted from the .cnv file */ \ |
| 430 | NULL, \ |
| 431 | \ |
| 432 | /* converter name for swaplfnl */ \ |
| 433 | NULL, \ |
| 434 | \ |
| 435 | /* extension data */ \ |
| 436 | NULL, \ |
| 437 | NULL \ |
| 438 | } |
| 439 | |
| 440 | enum { |
| 441 | MBCS_OPT_LENGTH_MASK=0x3f, |
| 442 | MBCS_OPT_NO_FROM_U=0x40, |
| 443 | /* |
| 444 | * If any of the following options bits are set, |
| 445 | * then the file must be rejected. |
| 446 | */ |
| 447 | MBCS_OPT_INCOMPATIBLE_MASK=0xffc0, |
| 448 | /* |
| 449 | * Remove bits from this mask as more options are recognized |
| 450 | * by all implementations that use this constant. |
| 451 | */ |
| 452 | MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80 |
| 453 | }; |
| 454 | |
| 455 | enum { |
| 456 | MBCS_HEADER_V4_LENGTH=8, |
| 457 | MBCS_HEADER_V5_MIN_LENGTH=9 |
| 458 | }; |
| 459 | |
| 460 | /** |
| 461 | * MBCS data header. See data format description above. |
| 462 | */ |
| 463 | typedef struct { |
| 464 | UVersionInfo version; |
| 465 | uint32_t countStates, |
| 466 | countToUFallbacks, |
| 467 | offsetToUCodeUnits, |
| 468 | offsetFromUTable, |
| 469 | offsetFromUBytes, |
| 470 | flags, |
| 471 | fromUBytesLength; |
| 472 | |
| 473 | /* new and required in version 5 */ |
| 474 | uint32_t options; |
| 475 | |
| 476 | /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */ |
| 477 | uint32_t fullStage2Length; /* number of 32-bit units */ |
| 478 | } _MBCSHeader; |
| 479 | |
| 480 | #define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 } |
| 481 | |
| 482 | /* |
| 483 | * This is a simple version of _MBCSGetNextUChar() that is used |
| 484 | * by other converter implementations. |
| 485 | * It only returns an "assigned" result if it consumes the entire input. |
| 486 | * It does not use state from the converter, nor error codes. |
| 487 | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
| 488 | * It handles conversion extensions but not GB 18030. |
| 489 | * |
| 490 | * Return value: |
| 491 | * U+fffe unassigned |
| 492 | * U+ffff illegal |
| 493 | * otherwise the Unicode code point |
| 494 | */ |
| 495 | U_CFUNC UChar32 |
| 496 | ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, |
| 497 | const char *source, int32_t length, |
| 498 | UBool useFallback); |
| 499 | |
| 500 | /** |
| 501 | * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. |
| 502 | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
| 503 | * It does not handle conversion extensions (_extToU()). |
| 504 | */ |
| 505 | U_CFUNC UChar32 |
| 506 | ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, |
| 507 | uint8_t b, UBool useFallback); |
| 508 | |
| 509 | /** |
| 510 | * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte. |
| 511 | * It works for single-byte, single-state codepages that only map |
| 512 | * to and from BMP code points, and it always |
| 513 | * returns fallback values. |
| 514 | */ |
| 515 | #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \ |
| 516 | (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)]) |
| 517 | |
| 518 | /** |
| 519 | * This is an internal function that allows other converter implementations |
| 520 | * to check whether a byte is a lead byte. |
| 521 | */ |
| 522 | U_CFUNC UBool |
| 523 | ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte); |
| 524 | |
| 525 | /** This is a macro version of _MBCSIsLeadByte(). */ |
| 526 | #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \ |
| 527 | (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)]) |
| 528 | |
| 529 | /* |
| 530 | * This is another simple conversion function for internal use by other |
| 531 | * conversion implementations. |
| 532 | * It does not use the converter state nor call callbacks. |
| 533 | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
| 534 | * It handles conversion extensions but not GB 18030. |
| 535 | * |
| 536 | * It converts one single Unicode code point into codepage bytes, encoded |
| 537 | * as one 32-bit value. The function returns the number of bytes in *pValue: |
| 538 | * 1..4 the number of bytes in *pValue |
| 539 | * 0 unassigned (*pValue undefined) |
| 540 | * -1 illegal (currently not used, *pValue undefined) |
| 541 | * |
| 542 | * *pValue will contain the resulting bytes with the last byte in bits 7..0, |
| 543 | * the second to last byte in bits 15..8, etc. |
| 544 | * Currently, the function assumes but does not check that 0<=c<=0x10ffff. |
| 545 | */ |
| 546 | U_CFUNC int32_t |
| 547 | ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, |
| 548 | UChar32 c, uint32_t *pValue, |
| 549 | UBool useFallback); |
| 550 | |
| 551 | /** |
| 552 | * This version of _MBCSFromUChar32() is optimized for single-byte codepages. |
| 553 | * It does not handle the EBCDIC swaplfnl option (set in UConverter). |
| 554 | * |
| 555 | * It returns the codepage byte for the code point, or -1 if it is unassigned. |
| 556 | */ |
| 557 | U_CFUNC int32_t |
| 558 | ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, |
| 559 | UChar32 c, |
| 560 | UBool useFallback); |
| 561 | |
| 562 | /** |
| 563 | * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but |
| 564 | * we cheat a little about the type, returning the old types if appropriate. |
| 565 | */ |
| 566 | U_CFUNC UConverterType |
| 567 | ucnv_MBCSGetType(const UConverter* converter); |
| 568 | |
| 569 | U_CFUNC void |
| 570 | ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, |
| 571 | UErrorCode *pErrorCode); |
| 572 | U_CFUNC void |
| 573 | ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, |
| 574 | UErrorCode *pErrorCode); |
| 575 | |
| 576 | /* |
| 577 | * Internal function returning a UnicodeSet for toUnicode() conversion. |
| 578 | * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. |
| 579 | * In the future, if we add support for fallback sets, this function |
| 580 | * needs to be updated. |
| 581 | * Handles extensions. |
| 582 | * Does not empty the set first. |
| 583 | */ |
| 584 | U_CFUNC void |
| 585 | ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, |
| 586 | const USetAdder *sa, |
| 587 | UConverterUnicodeSet which, |
| 588 | UErrorCode *pErrorCode); |
| 589 | |
| 590 | /* |
| 591 | * Same as ucnv_MBCSGetUnicodeSetForUnicode() but |
| 592 | * the set can be filtered by encoding scheme. |
| 593 | * Used by stateful converters which share regular conversion tables |
| 594 | * but only use a subset of their mappings. |
| 595 | */ |
| 596 | U_CFUNC void |
| 597 | ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, |
| 598 | const USetAdder *sa, |
| 599 | UConverterUnicodeSet which, |
| 600 | UConverterSetFilter filter, |
| 601 | UErrorCode *pErrorCode); |
| 602 | |
| 603 | #endif |
| 604 | |
| 605 | #endif |