whesse@chromium.org | e90029b | 2010-08-02 11:52:17 +0000 | [diff] [blame] | 1 | // Copyright 2007-2010 the V8 project authors. All rights reserved. |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 2 | // Redistribution and use in source and binary forms, with or without |
| 3 | // modification, are permitted provided that the following conditions are |
| 4 | // met: |
| 5 | // |
| 6 | // * Redistributions of source code must retain the above copyright |
| 7 | // notice, this list of conditions and the following disclaimer. |
| 8 | // * Redistributions in binary form must reproduce the above |
| 9 | // copyright notice, this list of conditions and the following |
| 10 | // disclaimer in the documentation and/or other materials provided |
| 11 | // with the distribution. |
| 12 | // * Neither the name of Google Inc. nor the names of its |
| 13 | // contributors may be used to endorse or promote products derived |
| 14 | // from this software without specific prior written permission. |
| 15 | // |
| 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
ager@chromium.org | 5ec4892 | 2009-05-05 07:25:34 +0000 | [diff] [blame] | 28 | #ifndef V8_UNICODE_INL_H_ |
| 29 | #define V8_UNICODE_INL_H_ |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 30 | |
| 31 | #include "unicode.h" |
| 32 | |
| 33 | namespace unibrow { |
| 34 | |
| 35 | template <class T, int s> bool Predicate<T, s>::get(uchar code_point) { |
| 36 | CacheEntry entry = entries_[code_point & kMask]; |
| 37 | if (entry.code_point_ == code_point) return entry.value_; |
| 38 | return CalculateValue(code_point); |
| 39 | } |
| 40 | |
| 41 | template <class T, int s> bool Predicate<T, s>::CalculateValue( |
| 42 | uchar code_point) { |
| 43 | bool result = T::Is(code_point); |
| 44 | entries_[code_point & kMask] = CacheEntry(code_point, result); |
| 45 | return result; |
| 46 | } |
| 47 | |
| 48 | template <class T, int s> int Mapping<T, s>::get(uchar c, uchar n, |
| 49 | uchar* result) { |
| 50 | CacheEntry entry = entries_[c & kMask]; |
| 51 | if (entry.code_point_ == c) { |
| 52 | if (entry.offset_ == 0) { |
| 53 | return 0; |
| 54 | } else { |
| 55 | result[0] = c + entry.offset_; |
| 56 | return 1; |
| 57 | } |
| 58 | } else { |
| 59 | return CalculateValue(c, n, result); |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | template <class T, int s> int Mapping<T, s>::CalculateValue(uchar c, uchar n, |
| 64 | uchar* result) { |
| 65 | bool allow_caching = true; |
| 66 | int length = T::Convert(c, n, result, &allow_caching); |
| 67 | if (allow_caching) { |
| 68 | if (length == 1) { |
| 69 | entries_[c & kMask] = CacheEntry(c, result[0] - c); |
| 70 | return 1; |
| 71 | } else { |
| 72 | entries_[c & kMask] = CacheEntry(c, 0); |
| 73 | return 0; |
| 74 | } |
| 75 | } else { |
| 76 | return length; |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | |
yangguo@chromium.org | 154ff99 | 2012-03-13 08:09:54 +0000 | [diff] [blame] | 81 | unsigned Utf8::Encode(char* str, uchar c, int previous) { |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 82 | static const int kMask = ~(1 << 6); |
| 83 | if (c <= kMaxOneByteChar) { |
| 84 | str[0] = c; |
| 85 | return 1; |
| 86 | } else if (c <= kMaxTwoByteChar) { |
| 87 | str[0] = 0xC0 | (c >> 6); |
| 88 | str[1] = 0x80 | (c & kMask); |
| 89 | return 2; |
| 90 | } else if (c <= kMaxThreeByteChar) { |
yangguo@chromium.org | 154ff99 | 2012-03-13 08:09:54 +0000 | [diff] [blame] | 91 | if (Utf16::IsTrailSurrogate(c) && |
| 92 | Utf16::IsLeadSurrogate(previous)) { |
| 93 | const int kUnmatchedSize = kSizeOfUnmatchedSurrogate; |
| 94 | return Encode(str - kUnmatchedSize, |
| 95 | Utf16::CombineSurrogatePair(previous, c), |
| 96 | Utf16::kNoPreviousCharacter) - kUnmatchedSize; |
| 97 | } |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 98 | str[0] = 0xE0 | (c >> 12); |
| 99 | str[1] = 0x80 | ((c >> 6) & kMask); |
| 100 | str[2] = 0x80 | (c & kMask); |
| 101 | return 3; |
| 102 | } else { |
| 103 | str[0] = 0xF0 | (c >> 18); |
| 104 | str[1] = 0x80 | ((c >> 12) & kMask); |
| 105 | str[2] = 0x80 | ((c >> 6) & kMask); |
| 106 | str[3] = 0x80 | (c & kMask); |
| 107 | return 4; |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | |
| 112 | uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) { |
| 113 | if (length <= 0) return kBadChar; |
| 114 | byte first = bytes[0]; |
| 115 | // Characters between 0000 and 0007F are encoded as a single character |
| 116 | if (first <= kMaxOneByteChar) { |
| 117 | *cursor += 1; |
| 118 | return first; |
| 119 | } |
| 120 | return CalculateValue(bytes, length, cursor); |
| 121 | } |
| 122 | |
yangguo@chromium.org | 154ff99 | 2012-03-13 08:09:54 +0000 | [diff] [blame] | 123 | unsigned Utf8::Length(uchar c, int previous) { |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 124 | if (c <= kMaxOneByteChar) { |
| 125 | return 1; |
| 126 | } else if (c <= kMaxTwoByteChar) { |
| 127 | return 2; |
| 128 | } else if (c <= kMaxThreeByteChar) { |
yangguo@chromium.org | 154ff99 | 2012-03-13 08:09:54 +0000 | [diff] [blame] | 129 | if (Utf16::IsTrailSurrogate(c) && |
| 130 | Utf16::IsLeadSurrogate(previous)) { |
| 131 | return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates; |
| 132 | } |
christian.plesner.hansen | 43d26ec | 2008-07-03 15:10:15 +0000 | [diff] [blame] | 133 | return 3; |
| 134 | } else { |
| 135 | return 4; |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | uchar CharacterStream::GetNext() { |
| 140 | uchar result = DecodeCharacter(buffer_, &cursor_); |
| 141 | if (remaining_ == 1) { |
| 142 | cursor_ = 0; |
| 143 | FillBuffer(); |
| 144 | } else { |
| 145 | remaining_--; |
| 146 | } |
| 147 | return result; |
| 148 | } |
| 149 | |
| 150 | #if __BYTE_ORDER == __LITTLE_ENDIAN |
| 151 | #define IF_LITTLE(expr) expr |
| 152 | #define IF_BIG(expr) ((void) 0) |
| 153 | #elif __BYTE_ORDER == __BIG_ENDIAN |
| 154 | #define IF_LITTLE(expr) ((void) 0) |
| 155 | #define IF_BIG(expr) expr |
| 156 | #else |
| 157 | #warning Unknown byte ordering |
| 158 | #endif |
| 159 | |
| 160 | bool CharacterStream::EncodeAsciiCharacter(uchar c, byte* buffer, |
| 161 | unsigned capacity, unsigned& offset) { |
| 162 | if (offset >= capacity) return false; |
| 163 | buffer[offset] = c; |
| 164 | offset += 1; |
| 165 | return true; |
| 166 | } |
| 167 | |
| 168 | bool CharacterStream::EncodeNonAsciiCharacter(uchar c, byte* buffer, |
| 169 | unsigned capacity, unsigned& offset) { |
| 170 | unsigned aligned = (offset + 0x3) & ~0x3; |
| 171 | if ((aligned + sizeof(uchar)) > capacity) |
| 172 | return false; |
| 173 | if (offset == aligned) { |
| 174 | IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = (c << 8) | 0x80); |
| 175 | IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c | (1 << 31)); |
| 176 | } else { |
| 177 | buffer[offset] = 0x80; |
| 178 | IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = c << 8); |
| 179 | IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c); |
| 180 | } |
| 181 | offset = aligned + sizeof(uchar); |
| 182 | return true; |
| 183 | } |
| 184 | |
| 185 | bool CharacterStream::EncodeCharacter(uchar c, byte* buffer, unsigned capacity, |
| 186 | unsigned& offset) { |
| 187 | if (c <= Utf8::kMaxOneByteChar) { |
| 188 | return EncodeAsciiCharacter(c, buffer, capacity, offset); |
| 189 | } else { |
| 190 | return EncodeNonAsciiCharacter(c, buffer, capacity, offset); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | uchar CharacterStream::DecodeCharacter(const byte* buffer, unsigned* offset) { |
| 195 | byte b = buffer[*offset]; |
| 196 | if (b <= Utf8::kMaxOneByteChar) { |
| 197 | (*offset)++; |
| 198 | return b; |
| 199 | } else { |
| 200 | unsigned aligned = (*offset + 0x3) & ~0x3; |
| 201 | *offset = aligned + sizeof(uchar); |
| 202 | IF_LITTLE(return *reinterpret_cast<const uchar*>(buffer + aligned) >> 8); |
| 203 | IF_BIG(return *reinterpret_cast<const uchar*>(buffer + aligned) & |
| 204 | ~(1 << 31)); |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | #undef IF_LITTLE |
| 209 | #undef IF_BIG |
| 210 | |
| 211 | template <class R, class I, unsigned s> |
| 212 | void InputBuffer<R, I, s>::FillBuffer() { |
| 213 | buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); |
| 214 | } |
| 215 | |
| 216 | template <class R, class I, unsigned s> |
| 217 | void InputBuffer<R, I, s>::Rewind() { |
| 218 | Reset(input_); |
| 219 | } |
| 220 | |
| 221 | template <class R, class I, unsigned s> |
| 222 | void InputBuffer<R, I, s>::Reset(unsigned position, I input) { |
| 223 | input_ = input; |
| 224 | remaining_ = 0; |
| 225 | cursor_ = 0; |
| 226 | offset_ = position; |
| 227 | buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); |
| 228 | } |
| 229 | |
| 230 | template <class R, class I, unsigned s> |
| 231 | void InputBuffer<R, I, s>::Reset(I input) { |
| 232 | Reset(0, input); |
| 233 | } |
| 234 | |
| 235 | template <class R, class I, unsigned s> |
| 236 | void InputBuffer<R, I, s>::Seek(unsigned position) { |
| 237 | offset_ = position; |
| 238 | buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); |
| 239 | } |
| 240 | |
| 241 | template <unsigned s> |
| 242 | Utf8InputBuffer<s>::Utf8InputBuffer(const char* data, unsigned length) |
| 243 | : InputBuffer<Utf8, Buffer<const char*>, s>(Buffer<const char*>(data, |
| 244 | length)) { |
| 245 | } |
| 246 | |
| 247 | } // namespace unibrow |
| 248 | |
ager@chromium.org | 5ec4892 | 2009-05-05 07:25:34 +0000 | [diff] [blame] | 249 | #endif // V8_UNICODE_INL_H_ |