Btk
unchecked.h
1 // Copyright 2006 Nemanja Trifunovic
2 
3 /*
4 Permission is hereby granted, free of charge, to any person or organization
5 obtaining a copy of the software and accompanying documentation covered by
6 this license (the "Software") to use, reproduce, display, distribute,
7 execute, and transmit the Software, and to prepare derivative works of the
8 Software, and to permit third-parties to whom the Software is furnished to
9 do so, all subject to the following:
10 
11 The copyright notices in the Software and this entire statement, including
12 the above license grant, this restriction and the following disclaimer,
13 must be included in all copies of the Software, in whole or in part, and
14 all derivative works of the Software, unless such copies or derivative
15 works are solely in the form of machine-executable object code generated by
16 a source language processor.
17 
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
21 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
22 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
23 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 DEALINGS IN THE SOFTWARE.
25 */
26 
27 
28 #ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
29 #define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
30 
31 #include "core.h"
32 
33 namespace utf8
34 {
35  namespace unchecked
36  {
37  template <typename octet_iterator>
38  octet_iterator append(uint32_t cp, octet_iterator result)
39  {
40  if (cp < 0x80) // one octet
41  *(result++) = static_cast<uint8_t>(cp);
42  else if (cp < 0x800) { // two octets
43  *(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
44  *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
45  }
46  else if (cp < 0x10000) { // three octets
47  *(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
48  *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
49  *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
50  }
51  else { // four octets
52  *(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
53  *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
54  *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
55  *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
56  }
57  return result;
58  }
59 
60  template <typename octet_iterator, typename output_iterator>
61  output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
62  {
63  while (start != end) {
64  octet_iterator sequence_start = start;
65  internal::utf_error err_code = utf8::internal::validate_next(start, end);
66  switch (err_code) {
67  case internal::UTF8_OK :
68  for (octet_iterator it = sequence_start; it != start; ++it)
69  *out++ = *it;
70  break;
71  case internal::NOT_ENOUGH_ROOM:
72  out = utf8::unchecked::append (replacement, out);
73  start = end;
74  break;
75  case internal::INVALID_LEAD:
76  out = utf8::unchecked::append (replacement, out);
77  ++start;
78  break;
79  case internal::INCOMPLETE_SEQUENCE:
80  case internal::OVERLONG_SEQUENCE:
81  case internal::INVALID_CODE_POINT:
82  out = utf8::unchecked::append (replacement, out);
83  ++start;
84  // just one replacement mark for the sequence
85  while (start != end && utf8::internal::is_trail(*start))
86  ++start;
87  break;
88  }
89  }
90  return out;
91  }
92 
93  template <typename octet_iterator, typename output_iterator>
94  inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
95  {
96  static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
97  return utf8::unchecked::replace_invalid(start, end, out, replacement_marker);
98  }
99 
100  template <typename octet_iterator>
101  uint32_t next(octet_iterator& it)
102  {
103  uint32_t cp = utf8::internal::mask8(*it);
104  typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
105  switch (length) {
106  case 1:
107  break;
108  case 2:
109  it++;
110  cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
111  break;
112  case 3:
113  ++it;
114  cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
115  ++it;
116  cp += (*it) & 0x3f;
117  break;
118  case 4:
119  ++it;
120  cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
121  ++it;
122  cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
123  ++it;
124  cp += (*it) & 0x3f;
125  break;
126  }
127  ++it;
128  return cp;
129  }
130 
131  template <typename octet_iterator>
132  uint32_t peek_next(octet_iterator it)
133  {
134  return utf8::unchecked::next(it);
135  }
136 
137  template <typename octet_iterator>
138  uint32_t prior(octet_iterator& it)
139  {
140  while (utf8::internal::is_trail(*(--it))) ;
141  octet_iterator temp = it;
142  return utf8::unchecked::next(temp);
143  }
144 
145  template <typename octet_iterator, typename distance_type>
146  void advance (octet_iterator& it, distance_type n)
147  {
148  const distance_type zero(0);
149  if (n < zero) {
150  // backward
151  for (distance_type i = n; i < zero; ++i)
152  utf8::unchecked::prior(it);
153  } else {
154  // forward
155  for (distance_type i = zero; i < n; ++i)
156  utf8::unchecked::next(it);
157  }
158  }
159 
160  template <typename octet_iterator>
161  typename std::iterator_traits<octet_iterator>::difference_type
162  distance (octet_iterator first, octet_iterator last)
163  {
164  typename std::iterator_traits<octet_iterator>::difference_type dist;
165  for (dist = 0; first < last; ++dist)
166  utf8::unchecked::next(first);
167  return dist;
168  }
169 
170  template <typename u16bit_iterator, typename octet_iterator>
171  octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
172  {
173  while (start != end) {
174  uint32_t cp = utf8::internal::mask16(*start++);
175  // Take care of surrogate pairs first
176  if (utf8::internal::is_lead_surrogate(cp)) {
177  uint32_t trail_surrogate = utf8::internal::mask16(*start++);
178  cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
179  }
180  result = utf8::unchecked::append(cp, result);
181  }
182  return result;
183  }
184 
185  template <typename u16bit_iterator, typename octet_iterator>
186  u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
187  {
188  while (start < end) {
189  uint32_t cp = utf8::unchecked::next(start);
190  if (cp > 0xffff) { //make a surrogate pair
191  *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
192  *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
193  }
194  else
195  *result++ = static_cast<uint16_t>(cp);
196  }
197  return result;
198  }
199 
200  template <typename octet_iterator, typename u32bit_iterator>
201  octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
202  {
203  while (start != end)
204  result = utf8::unchecked::append(*(start++), result);
205 
206  return result;
207  }
208 
209  template <typename octet_iterator, typename u32bit_iterator>
210  u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
211  {
212  while (start < end)
213  (*result++) = utf8::unchecked::next(start);
214 
215  return result;
216  }
217 
218  // The iterator class
219  template <typename octet_iterator>
220  class iterator {
221  octet_iterator it;
222  public:
223  typedef uint32_t value_type;
224  typedef uint32_t* pointer;
225  typedef uint32_t& reference;
226  typedef std::ptrdiff_t difference_type;
227  typedef std::bidirectional_iterator_tag iterator_category;
228  iterator () {}
229  explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
230  // the default "big three" are OK
231  octet_iterator base () const { return it; }
232  uint32_t operator * () const
233  {
234  octet_iterator temp = it;
235  return utf8::unchecked::next(temp);
236  }
237  bool operator == (const iterator& rhs) const
238  {
239  return (it == rhs.it);
240  }
241  bool operator != (const iterator& rhs) const
242  {
243  return !(operator == (rhs));
244  }
245  iterator& operator ++ ()
246  {
247  ::std::advance(it, utf8::internal::sequence_length(it));
248  return *this;
249  }
250  iterator operator ++ (int)
251  {
252  iterator temp = *this;
253  ::std::advance(it, utf8::internal::sequence_length(it));
254  return temp;
255  }
256  iterator& operator -- ()
257  {
258  utf8::unchecked::prior(it);
259  return *this;
260  }
261  iterator operator -- (int)
262  {
263  iterator temp = *this;
264  utf8::unchecked::prior(it);
265  return temp;
266  }
267  }; // class iterator
268 
269  } // namespace utf8::unchecked
270 } // namespace utf8
271 
272 
273 #endif // header guard
274 
utf8::unchecked::iterator
Definition: unchecked.h:220
utf8
Definition: checked.h:34