Line data Source code
1 : /* nfkc.c --- Unicode normalization utilities. 2 : Copyright (C) 2002-2020 Simon Josefsson 3 : 4 : This file is part of GNU Libidn. 5 : 6 : GNU Libidn is free software: you can redistribute it and/or 7 : modify it under the terms of either: 8 : 9 : * the GNU Lesser General Public License as published by the Free 10 : Software Foundation; either version 3 of the License, or (at 11 : your option) any later version. 12 : 13 : or 14 : 15 : * the GNU General Public License as published by the Free 16 : Software Foundation; either version 2 of the License, or (at 17 : your option) any later version. 18 : 19 : or both in parallel, as here. 20 : 21 : GNU Libidn is distributed in the hope that it will be useful, 22 : but WITHOUT ANY WARRANTY; without even the implied warranty of 23 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 24 : General Public License for more details. 25 : 26 : You should have received copies of the GNU General Public License and 27 : the GNU Lesser General Public License along with this program. If 28 : not, see <http://www.gnu.org/licenses/>. */ 29 : 30 : #ifdef HAVE_CONFIG_H 31 : # include "config.h" 32 : #endif 33 : 34 : #include <stdlib.h> 35 : #include <string.h> 36 : 37 : #include "stringprep.h" 38 : 39 : /* Hacks to make syncing with GLIB code easier. */ 40 : #define gboolean int 41 : #define gchar char 42 : #define guchar unsigned char 43 : #define glong long 44 : #define gint int 45 : #define guint unsigned int 46 : #define gushort unsigned short 47 : #define gint16 int16_t 48 : #define guint16 uint16_t 49 : #define gunichar uint32_t 50 : #define gsize size_t 51 : #define gssize ssize_t 52 : #define g_malloc malloc 53 : #define g_free free 54 : #define g_return_val_if_fail(expr,val) { \ 55 : if (!(expr)) \ 56 : return (val); \ 57 : } 58 : 59 : /* Code from GLIB gmacros.h starts here. */ 60 : 61 : /* GLIB - Library of useful routines for C programming 62 : * Copyright (C) 1995-1997 Peter Mattis, Spencer Kimball and Josh MacDonald 63 : * 64 : * This library is free software; you can redistribute it and/or 65 : * modify it under the terms of the GNU Lesser General Public 66 : * License as published by the Free Software Foundation; either 67 : * version 2 of the License, or (at your option) any later version. 68 : * 69 : * This library is distributed in the hope that it will be useful, 70 : * but WITHOUT ANY WARRANTY; without even the implied warranty of 71 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 72 : * Lesser General Public License for more details. 73 : * 74 : * You should have received a copy of the GNU Lesser General Public 75 : * License along with this library; if not, write to the 76 : * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 77 : * Boston, MA 02111-1307, USA. 78 : */ 79 : 80 : #ifndef FALSE 81 : # define FALSE (0) 82 : #endif 83 : 84 : #ifndef TRUE 85 : # define TRUE (!FALSE) 86 : #endif 87 : 88 : #define G_N_ELEMENTS(arr) (sizeof (arr) / sizeof ((arr)[0])) 89 : 90 : #define G_UNLIKELY(expr) (expr) 91 : 92 : /* Code from GLIB gunicode.h starts here. */ 93 : 94 : /* gunicode.h - Unicode manipulation functions 95 : * 96 : * Copyright (C) 1999, 2000 Tom Tromey 97 : * Copyright 2000, 2005 Red Hat, Inc. 98 : * 99 : * The Gnome Library is free software; you can redistribute it and/or 100 : * modify it under the terms of the GNU Lesser General Public License as 101 : * published by the Free Software Foundation; either version 2 of the 102 : * License, or (at your option) any later version. 103 : * 104 : * The Gnome Library is distributed in the hope that it will be useful, 105 : * but WITHOUT ANY WARRANTY; without even the implied warranty of 106 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 107 : * Lesser General Public License for more details. 108 : * 109 : * You should have received a copy of the GNU Lesser General Public 110 : * License along with the Gnome Library; see the file COPYING.LIB. If not, 111 : * write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, 112 : * Boston, MA 02111-1307, USA. 113 : */ 114 : 115 : typedef enum 116 : { 117 : G_NORMALIZE_DEFAULT, 118 : G_NORMALIZE_NFD = G_NORMALIZE_DEFAULT, 119 : G_NORMALIZE_DEFAULT_COMPOSE, 120 : G_NORMALIZE_NFC = G_NORMALIZE_DEFAULT_COMPOSE, 121 : G_NORMALIZE_ALL, 122 : G_NORMALIZE_NFKD = G_NORMALIZE_ALL, 123 : G_NORMALIZE_ALL_COMPOSE, 124 : G_NORMALIZE_NFKC = G_NORMALIZE_ALL_COMPOSE 125 : } 126 : GNormalizeMode; 127 : 128 : #define g_utf8_next_char(p) ((p) + g_utf8_skip[*(const guchar *)(p)]) 129 : 130 : /* Code from GLIB gutf8.c starts here. */ 131 : 132 : /* gutf8.c - Operations on UTF-8 strings. 133 : * 134 : * Copyright (C) 1999 Tom Tromey 135 : * Copyright (C) 2000 Red Hat, Inc. 136 : * 137 : * This library is free software; you can redistribute it and/or 138 : * modify it under the terms of the GNU Lesser General Public 139 : * License as published by the Free Software Foundation; either 140 : * version 2 of the License, or (at your option) any later version. 141 : * 142 : * This library is distributed in the hope that it will be useful, 143 : * but WITHOUT ANY WARRANTY; without even the implied warranty of 144 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 145 : * Lesser General Public License for more details. 146 : * 147 : * You should have received a copy of the GNU Lesser General Public 148 : * License along with this library; if not, write to the 149 : * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 150 : * Boston, MA 02111-1307, USA. 151 : */ 152 : 153 : #define UTF8_COMPUTE(Char, Mask, Len) \ 154 : if (Char < 128) \ 155 : { \ 156 : Len = 1; \ 157 : Mask = 0x7f; \ 158 : } \ 159 : else if ((Char & 0xe0) == 0xc0) \ 160 : { \ 161 : Len = 2; \ 162 : Mask = 0x1f; \ 163 : } \ 164 : else if ((Char & 0xf0) == 0xe0) \ 165 : { \ 166 : Len = 3; \ 167 : Mask = 0x0f; \ 168 : } \ 169 : else if ((Char & 0xf8) == 0xf0) \ 170 : { \ 171 : Len = 4; \ 172 : Mask = 0x07; \ 173 : } \ 174 : else if ((Char & 0xfc) == 0xf8) \ 175 : { \ 176 : Len = 5; \ 177 : Mask = 0x03; \ 178 : } \ 179 : else if ((Char & 0xfe) == 0xfc) \ 180 : { \ 181 : Len = 6; \ 182 : Mask = 0x01; \ 183 : } \ 184 : else \ 185 : Len = -1; 186 : 187 : #define UTF8_LENGTH(Char) \ 188 : ((Char) < 0x80 ? 1 : \ 189 : ((Char) < 0x800 ? 2 : \ 190 : ((Char) < 0x10000 ? 3 : \ 191 : ((Char) < 0x200000 ? 4 : \ 192 : ((Char) < 0x4000000 ? 5 : 6))))) 193 : 194 : #define UTF8_GET(Result, Chars, Count, Mask, Len) \ 195 : (Result) = (Chars)[0] & (Mask); \ 196 : for ((Count) = 1; (Count) < (Len); ++(Count)) \ 197 : { \ 198 : if (((Chars)[(Count)] & 0xc0) != 0x80) \ 199 : { \ 200 : (Result) = -1; \ 201 : break; \ 202 : } \ 203 : (Result) <<= 6; \ 204 : (Result) |= ((Chars)[(Count)] & 0x3f); \ 205 : } 206 : 207 : static const gchar utf8_skip_data[256] = { 208 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 209 : 1, 1, 1, 1, 1, 1, 1, 210 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 211 : 1, 1, 1, 1, 1, 1, 1, 212 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 213 : 1, 1, 1, 1, 1, 1, 1, 214 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 215 : 1, 1, 1, 1, 1, 1, 1, 216 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 217 : 1, 1, 1, 1, 1, 1, 1, 218 : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 219 : 1, 1, 1, 1, 1, 1, 1, 220 : 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 221 : 2, 2, 2, 2, 2, 2, 2, 222 : 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 223 : 5, 5, 5, 6, 6, 1, 1 224 : }; 225 : 226 : static const gchar *const g_utf8_skip = utf8_skip_data; 227 : 228 : /* 229 : * g_utf8_strlen: 230 : * @p: pointer to the start of a UTF-8 encoded string 231 : * @max: the maximum number of bytes to examine. If @max 232 : * is less than 0, then the string is assumed to be 233 : * nul-terminated. If @max is 0, @p will not be examined and 234 : * may be %NULL. 235 : * 236 : * Computes the length of the string in characters, not including 237 : * the terminating nul character. 238 : * 239 : * Return value: the length of the string in characters 240 : **/ 241 : static glong 242 237358 : g_utf8_strlen (const gchar * p) 243 : { 244 237358 : glong len = 0; 245 : 246 237358 : g_return_val_if_fail (p != NULL, 0); 247 : 248 1460478 : while (*p) 249 : { 250 1223120 : p = g_utf8_next_char (p); 251 1223120 : ++len; 252 : } 253 : 254 237358 : return len; 255 : } 256 : 257 : /* 258 : * g_utf8_get_char: 259 : * @p: a pointer to Unicode character encoded as UTF-8 260 : * 261 : * Converts a sequence of bytes encoded as UTF-8 to a Unicode character. 262 : * If @p does not point to a valid UTF-8 encoded character, results are 263 : * undefined. If you are not sure that the bytes are complete 264 : * valid Unicode characters, you should use g_utf8_get_char_validated() 265 : * instead. 266 : * 267 : * Return value: the resulting character 268 : **/ 269 : static gunichar 270 2322340 : g_utf8_get_char (const gchar * p) 271 : { 272 2322340 : int i, mask = 0, len; 273 : gunichar result; 274 2322340 : unsigned char c = (unsigned char) *p; 275 : 276 2322340 : UTF8_COMPUTE (c, mask, len); 277 2322340 : if (len == -1) 278 16 : return (gunichar) - 1; 279 5152591 : UTF8_GET (result, p, i, mask, len); 280 : 281 2322324 : return result; 282 : } 283 : 284 : /* 285 : * g_unichar_to_utf8: 286 : * @c: a Unicode character code 287 : * @outbuf: output buffer, must have at least 6 bytes of space. 288 : * If %NULL, the length will be computed and returned 289 : * and nothing will be written to @outbuf. 290 : * 291 : * Converts a single character to UTF-8. 292 : * 293 : * Return value: number of bytes written 294 : **/ 295 : static int 296 1544635 : g_unichar_to_utf8 (gunichar c, gchar * outbuf) 297 : { 298 : /* If this gets modified, also update the copy in g_string_insert_unichar() */ 299 1544635 : guint len = 0; 300 : int first; 301 : int i; 302 : 303 1544635 : if (c < 0x80) 304 : { 305 883799 : first = 0; 306 883799 : len = 1; 307 : } 308 660836 : else if (c < 0x800) 309 : { 310 181377 : first = 0xc0; 311 181377 : len = 2; 312 : } 313 479459 : else if (c < 0x10000) 314 : { 315 434178 : first = 0xe0; 316 434178 : len = 3; 317 : } 318 45281 : else if (c < 0x200000) 319 : { 320 20222 : first = 0xf0; 321 20222 : len = 4; 322 : } 323 25059 : else if (c < 0x4000000) 324 : { 325 460 : first = 0xf8; 326 460 : len = 5; 327 : } 328 : else 329 : { 330 24599 : first = 0xfc; 331 24599 : len = 6; 332 : } 333 : 334 1544635 : if (outbuf) 335 : { 336 2779869 : for (i = len - 1; i > 0; --i) 337 : { 338 1235234 : outbuf[i] = (c & 0x3f) | 0x80; 339 1235234 : c >>= 6; 340 : } 341 1544635 : outbuf[0] = c | first; 342 : } 343 : 344 1544635 : return len; 345 : } 346 : 347 : /* 348 : * g_utf8_to_ucs4_fast: 349 : * @str: a UTF-8 encoded string 350 : * @len: the maximum length of @str to use, in bytes. If @len < 0, 351 : * then the string is nul-terminated. 352 : * @items_written: location to store the number of characters in the 353 : * result, or %NULL. 354 : * 355 : * Convert a string from UTF-8 to a 32-bit fixed width 356 : * representation as UCS-4, assuming valid UTF-8 input. 357 : * This function is roughly twice as fast as g_utf8_to_ucs4() 358 : * but does no error checking on the input. A trailing 0 character 359 : * will be added to the string after the converted text. 360 : * 361 : * Return value: a pointer to a newly allocated UCS-4 string. 362 : * This value must be freed with g_free(). 363 : **/ 364 : static gunichar * 365 73814 : g_utf8_to_ucs4_fast (const gchar * str, glong len, glong * items_written) 366 : { 367 : gunichar *result; 368 : gsize n_chars, i; 369 : const gchar *p; 370 : 371 73814 : g_return_val_if_fail (str != NULL, NULL); 372 : 373 73814 : p = str; 374 73814 : n_chars = 0; 375 73814 : if (len < 0) 376 : { 377 1140777 : while (*p) 378 : { 379 1066963 : p = g_utf8_next_char (p); 380 1066963 : ++n_chars; 381 : } 382 : } 383 : else 384 : { 385 0 : while (p < str + len && *p) 386 : { 387 0 : p = g_utf8_next_char (p); 388 0 : ++n_chars; 389 : } 390 : } 391 : 392 73814 : result = g_malloc (sizeof (gunichar) * (n_chars + 1)); 393 73814 : if (!result) 394 0 : return NULL; 395 : 396 73814 : p = str; 397 1140777 : for (i = 0; i < n_chars; i++) 398 : { 399 1066963 : gunichar wc = (guchar) * p++; 400 : 401 1066963 : if (wc < 0x80) 402 : { 403 606540 : result[i] = wc; 404 : } 405 : else 406 : { 407 460423 : gunichar mask = 0x40; 408 : 409 460423 : if (G_UNLIKELY ((wc & mask) == 0)) 410 : { 411 : /* It's an out-of-sequence 10xxxxxxx byte. 412 : * Rather than making an ugly hash of this and the next byte 413 : * and overrunning the buffer, it's more useful to treat it 414 : * with a replacement character */ 415 0 : result[i] = 0xfffd; 416 0 : continue; 417 : } 418 : 419 : do 420 : { 421 822646 : wc <<= 6; 422 822646 : wc |= (guchar) (*p++) & 0x3f; 423 822646 : mask <<= 5; 424 : } 425 822646 : while ((wc & mask) != 0); 426 : 427 460423 : wc &= mask - 1; 428 : 429 460423 : result[i] = wc; 430 : } 431 : } 432 73814 : result[i] = 0; 433 : 434 73814 : if (items_written) 435 61919 : *items_written = i; 436 : 437 73814 : return result; 438 : } 439 : 440 : /* 441 : * g_ucs4_to_utf8: 442 : * @str: a UCS-4 encoded string 443 : * @len: the maximum length (number of characters) of @str to use. 444 : * If @len < 0, then the string is nul-terminated. 445 : * @items_read: location to store number of characters read, or %NULL. 446 : * @items_written: location to store number of bytes written or %NULL. 447 : * The value here stored does not include the trailing 0 448 : * byte. 449 : * @error: location to store the error occurring, or %NULL to ignore 450 : * errors. Any of the errors in #GConvertError other than 451 : * %G_CONVERT_ERROR_NO_CONVERSION may occur. 452 : * 453 : * Convert a string from a 32-bit fixed width representation as UCS-4. 454 : * to UTF-8. The result will be terminated with a 0 byte. 455 : * 456 : * Return value: a pointer to a newly allocated UTF-8 string. 457 : * This value must be freed with g_free(). If an 458 : * error occurs, %NULL will be returned and 459 : * @error set. In that case, @items_read will be 460 : * set to the position of the first invalid input 461 : * character. 462 : **/ 463 : static gchar * 464 138364 : g_ucs4_to_utf8 (const gunichar * str, 465 : glong len, glong * items_read, glong * items_written) 466 : { 467 : gint result_length; 468 138364 : gchar *result = NULL; 469 : gchar *p; 470 : gint i; 471 : 472 138364 : result_length = 0; 473 1687002 : for (i = 0; len < 0 || i < len; i++) 474 : { 475 1552845 : if (!str[i]) 476 3059 : break; 477 : 478 1549786 : if (str[i] >= 0x80000000) 479 1148 : goto err_out; 480 : 481 1548638 : result_length += UTF8_LENGTH (str[i]); 482 : } 483 : 484 137216 : result = g_malloc (result_length + 1); 485 137216 : if (!result) 486 0 : return NULL; 487 137216 : p = result; 488 : 489 137216 : i = 0; 490 1681487 : while (p < result + result_length) 491 1544271 : p += g_unichar_to_utf8 (str[i++], p); 492 : 493 137216 : *p = '0円'; 494 : 495 137216 : if (items_written) 496 9 : *items_written = p - result; 497 : 498 137207 : err_out: 499 138364 : if (items_read) 500 9 : *items_read = i; 501 : 502 138364 : return result; 503 : } 504 : 505 : /* Code from GLIB gunidecomp.c starts here. */ 506 : 507 : /* decomp.c - Character decomposition. 508 : * 509 : * Copyright (C) 1999, 2000 Tom Tromey 510 : * Copyright 2000 Red Hat, Inc. 511 : * 512 : * The Gnome Library is free software; you can redistribute it and/or 513 : * modify it under the terms of the GNU Lesser General Public License as 514 : * published by the Free Software Foundation; either version 2 of the 515 : * License, or (at your option) any later version. 516 : * 517 : * The Gnome Library is distributed in the hope that it will be useful, 518 : * but WITHOUT ANY WARRANTY; without even the implied warranty of 519 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 520 : * Lesser General Public License for more details. 521 : * 522 : * You should have received a copy of the GNU Lesser General Public 523 : * License along with the Gnome Library; see the file COPYING.LIB. If not, 524 : * write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, 525 : * Boston, MA 02111-1307, USA. 526 : */ 527 : 528 : #include "gunidecomp.h" 529 : #include "gunicomp.h" 530 : 531 : #define CC_PART1(Page, Char) \ 532 : ((combining_class_table_part1[Page] >= G_UNICODE_MAX_TABLE_INDEX) \ 533 : ? (combining_class_table_part1[Page] - G_UNICODE_MAX_TABLE_INDEX) \ 534 : : (cclass_data[combining_class_table_part1[Page]][Char])) 535 : 536 : #define CC_PART2(Page, Char) \ 537 : ((combining_class_table_part2[Page] >= G_UNICODE_MAX_TABLE_INDEX) \ 538 : ? (combining_class_table_part2[Page] - G_UNICODE_MAX_TABLE_INDEX) \ 539 : : (cclass_data[combining_class_table_part2[Page]][Char])) 540 : 541 : #define COMBINING_CLASS(Char) \ 542 : (((Char) <= G_UNICODE_LAST_CHAR_PART1) \ 543 : ? CC_PART1 ((Char) >> 8, (Char) & 0xff) \ 544 : : (((Char) >= 0xe0000 && (Char) <= G_UNICODE_LAST_CHAR) \ 545 : ? CC_PART2 (((Char) - 0xe0000) >> 8, (Char) & 0xff) \ 546 : : 0)) 547 : 548 : /* constants for hangul syllable [de]composition */ 549 : #define SBase 0xAC00 550 : #define LBase 0x1100 551 : #define VBase 0x1161 552 : #define TBase 0x11A7 553 : #define LCount 19 554 : #define VCount 21 555 : #define TCount 28 556 : #define NCount (VCount * TCount) 557 : #define SCount (LCount * NCount) 558 : 559 : /* 560 : * g_unicode_canonical_ordering: 561 : * @string: a UCS-4 encoded string. 562 : * @len: the maximum length of @string to use. 563 : * 564 : * Computes the canonical ordering of a string in-place. 565 : * This rearranges decomposed characters in the string 566 : * according to their combining classes. See the Unicode 567 : * manual for more information. 568 : **/ 569 : static void 570 576651 : g_unicode_canonical_ordering (gunichar * string, gsize len) 571 : { 572 : gsize i; 573 576651 : int swap = 1; 574 : 575 1162192 : while (swap) 576 : { 577 : int last; 578 585541 : swap = 0; 579 585541 : last = COMBINING_CLASS (string[0]); 580 3129202 : for (i = 0; i < len - 1; ++i) 581 : { 582 2543661 : int next = COMBINING_CLASS (string[i + 1]); 583 2543661 : if (next != 0 && last > next) 584 : { 585 : gsize j; 586 : /* Percolate item leftward through string. */ 587 33974 : for (j = i + 1; j > 0; --j) 588 : { 589 : gunichar t; 590 33305 : if (COMBINING_CLASS (string[j - 1]) <= next) 591 13690 : break; 592 19615 : t = string[j]; 593 19615 : string[j] = string[j - 1]; 594 19615 : string[j - 1] = t; 595 19615 : swap = 1; 596 : } 597 : /* We're re-entering the loop looking at the old 598 : character again. */ 599 14359 : next = last; 600 : } 601 2543661 : last = next; 602 : } 603 : } 604 576651 : } 605 : 606 : /* http://www.unicode.org/unicode/reports/tr15/#Hangul 607 : * r should be null or have sufficient space. Calling with r == NULL will 608 : * only calculate the result_len; however, a buffer with space for three 609 : * characters will always be big enough. */ 610 : static void 611 13126 : decompose_hangul (gunichar s, gunichar * r, gsize * result_len) 612 : { 613 13126 : gint SIndex = s - SBase; 614 13126 : gint TIndex = SIndex % TCount; 615 : 616 13126 : if (r) 617 : { 618 6563 : r[0] = LBase + SIndex / NCount; 619 6563 : r[1] = VBase + (SIndex % NCount) / TCount; 620 : } 621 : 622 13126 : if (TIndex) 623 : { 624 7320 : if (r) 625 3660 : r[2] = TBase + TIndex; 626 7320 : *result_len = 3; 627 : } 628 : else 629 5806 : *result_len = 2; 630 13126 : } 631 : 632 : /* returns a pointer to a null-terminated UTF-8 string */ 633 : static const gchar * 634 1085730 : find_decomposition (gunichar ch, gboolean compat) 635 : { 636 1085730 : int start = 0; 637 1085730 : int end = G_N_ELEMENTS (decomp_table); 638 : 639 1085730 : if (ch >= decomp_table[start].ch && ch <= decomp_table[end - 1].ch) 640 : { 641 : while (TRUE) 642 6672056 : { 643 7287978 : int half = (start + end) / 2; 644 7287978 : if (ch == decomp_table[half].ch) 645 : { 646 : int offset; 647 : 648 474716 : if (compat) 649 : { 650 474716 : offset = decomp_table[half].compat_offset; 651 474716 : if (offset == G_UNICODE_NOT_PRESENT_OFFSET) 652 36212 : offset = decomp_table[half].canon_offset; 653 : } 654 : else 655 : { 656 0 : offset = decomp_table[half].canon_offset; 657 0 : if (offset == G_UNICODE_NOT_PRESENT_OFFSET) 658 0 : return NULL; 659 : } 660 : 661 474716 : return &(decomp_expansion_string[offset]); 662 : } 663 6813262 : else if (half == start) 664 141206 : break; 665 6672056 : else if (ch > decomp_table[half].ch) 666 3084608 : start = half; 667 : else 668 3587448 : end = half; 669 : } 670 : } 671 : 672 611014 : return NULL; 673 : } 674 : 675 : /* L,V => LV and LV,T => LVT */ 676 : static gboolean 677 1483980 : combine_hangul (gunichar a, gunichar b, gunichar * result) 678 : { 679 1483980 : if (a >= LBase && a < LCount + LBase && b >= VBase && b < VCount + VBase) 680 : { 681 7077 : gint LIndex = a - LBase; 682 7077 : gint VIndex = b - VBase; 683 : 684 7077 : *result = SBase + (LIndex * VCount + VIndex) * TCount; 685 7077 : return TRUE; 686 : } 687 : 688 1476903 : if (a >= SBase && a < SCount + SBase && b > TBase && b < TCount + TBase) 689 : { 690 3661 : gint SIndex = a - SBase; 691 : 692 3661 : if ((SIndex % TCount) == 0) 693 : { 694 3661 : gint TIndex = b - TBase; 695 : 696 3661 : *result = a + TIndex; 697 3661 : return TRUE; 698 : } 699 : } 700 : 701 1473242 : return FALSE; 702 : } 703 : 704 : #define CI(Page, Char) \ 705 : ((compose_table[Page] >= G_UNICODE_MAX_TABLE_INDEX) \ 706 : ? (compose_table[Page] - G_UNICODE_MAX_TABLE_INDEX) \ 707 : : (compose_data[compose_table[Page]][Char])) 708 : 709 : #define COMPOSE_INDEX(Char) \ 710 : (((Char >> 8) > (COMPOSE_TABLE_LAST)) ? 0 : CI((Char) >> 8, (Char) & 0xff)) 711 : 712 : static gboolean 713 1483980 : combine (gunichar a, gunichar b, gunichar * result) 714 : { 715 : gushort index_a, index_b; 716 : 717 1483980 : if (combine_hangul (a, b, result)) 718 10738 : return TRUE; 719 : 720 1473242 : index_a = COMPOSE_INDEX (a); 721 : 722 1473242 : if (index_a >= COMPOSE_FIRST_SINGLE_START && index_a < COMPOSE_SECOND_START) 723 : { 724 177288 : if (b == compose_first_single[index_a - COMPOSE_FIRST_SINGLE_START][0]) 725 : { 726 30806 : *result = 727 30806 : compose_first_single[index_a - COMPOSE_FIRST_SINGLE_START][1]; 728 30806 : return TRUE; 729 : } 730 : else 731 146482 : return FALSE; 732 : } 733 : 734 1295954 : index_b = COMPOSE_INDEX (b); 735 : 736 1295954 : if (index_b >= COMPOSE_SECOND_SINGLE_START) 737 : { 738 10403 : if (a == 739 10403 : compose_second_single[index_b - COMPOSE_SECOND_SINGLE_START][0]) 740 : { 741 8837 : *result = 742 8837 : compose_second_single[index_b - COMPOSE_SECOND_SINGLE_START][1]; 743 8837 : return TRUE; 744 : } 745 : else 746 1566 : return FALSE; 747 : } 748 : 749 1285551 : if (index_a >= COMPOSE_FIRST_START && index_a < COMPOSE_FIRST_SINGLE_START 750 240731 : && index_b >= COMPOSE_SECOND_START 751 37700 : && index_b < COMPOSE_SECOND_SINGLE_START) 752 : { 753 37700 : gunichar res = 754 37700 : compose_array[index_a - COMPOSE_FIRST_START][index_b - 755 : COMPOSE_SECOND_START]; 756 : 757 37700 : if (res) 758 : { 759 35402 : *result = res; 760 35402 : return TRUE; 761 : } 762 : } 763 : 764 1250149 : return FALSE; 765 : } 766 : 767 : static gunichar * 768 54130 : _g_utf8_normalize_wc (const gchar * str, gssize max_len, GNormalizeMode mode) 769 : { 770 : gsize n_wc; 771 : gunichar *wc_buffer; 772 : const char *p; 773 : gsize last_start; 774 54130 : gboolean do_compat = (mode == G_NORMALIZE_NFKC || mode == G_NORMALIZE_NFKD); 775 54130 : gboolean do_compose = (mode == G_NORMALIZE_NFC || mode == G_NORMALIZE_NFKC); 776 : 777 54130 : n_wc = 0; 778 54130 : p = str; 779 603558 : while ((max_len < 0 || p < str + max_len) && *p) 780 : { 781 : const gchar *decomp; 782 549428 : gunichar wc = g_utf8_get_char (p); 783 : 784 549428 : if (wc >= SBase && wc < SBase + SCount) 785 6563 : { 786 : gsize result_len; 787 6563 : decompose_hangul (wc, NULL, &result_len); 788 6563 : n_wc += result_len; 789 : } 790 : else 791 : { 792 542865 : decomp = find_decomposition (wc, do_compat); 793 : 794 542865 : if (decomp) 795 237358 : n_wc += g_utf8_strlen (decomp); 796 : else 797 305507 : n_wc++; 798 : } 799 : 800 549428 : p = g_utf8_next_char (p); 801 : } 802 : 803 54130 : wc_buffer = g_malloc (sizeof (gunichar) * (n_wc + 1)); 804 54130 : if (!wc_buffer) 805 0 : return NULL; 806 : 807 54130 : last_start = 0; 808 54130 : n_wc = 0; 809 54130 : p = str; 810 603558 : while ((max_len < 0 || p < str + max_len) && *p) 811 : { 812 549428 : gunichar wc = g_utf8_get_char (p); 813 : const gchar *decomp; 814 : int cc; 815 549428 : gsize old_n_wc = n_wc; 816 : 817 549428 : if (wc >= SBase && wc < SBase + SCount) 818 6563 : { 819 : gsize result_len; 820 6563 : decompose_hangul (wc, wc_buffer + n_wc, &result_len); 821 6563 : n_wc += result_len; 822 : } 823 : else 824 : { 825 542865 : decomp = find_decomposition (wc, do_compat); 826 : 827 542865 : if (decomp) 828 : { 829 : const char *pd; 830 1460478 : for (pd = decomp; *pd != '0円'; pd = g_utf8_next_char (pd)) 831 1223120 : wc_buffer[n_wc++] = g_utf8_get_char (pd); 832 : } 833 : else 834 305507 : wc_buffer[n_wc++] = wc; 835 : } 836 : 837 549428 : if (n_wc > 0) 838 : { 839 549428 : cc = COMBINING_CLASS (wc_buffer[old_n_wc]); 840 : 841 549428 : if (cc == 0) 842 : { 843 523684 : g_unicode_canonical_ordering (wc_buffer + last_start, 844 : n_wc - last_start); 845 523684 : last_start = old_n_wc; 846 : } 847 : } 848 : 849 549428 : p = g_utf8_next_char (p); 850 : } 851 : 852 54130 : if (n_wc > 0) 853 : { 854 52967 : g_unicode_canonical_ordering (wc_buffer + last_start, 855 : n_wc - last_start); 856 : /* dead assignment: last_start = n_wc; */ 857 : } 858 : 859 54130 : wc_buffer[n_wc] = 0; 860 : 861 : /* All decomposed and reordered */ 862 : 863 54130 : if (do_compose && n_wc > 0) 864 : { 865 : gsize i, j; 866 52967 : int last_cc = 0; 867 52967 : last_start = 0; 868 : 869 1598380 : for (i = 0; i < n_wc; i++) 870 : { 871 1545413 : int cc = COMBINING_CLASS (wc_buffer[i]); 872 : 873 1545413 : if (i > 0 && 874 1514264 : (last_cc == 0 || last_cc != cc) && 875 1483980 : combine (wc_buffer[last_start], wc_buffer[i], 876 1483980 : &wc_buffer[last_start])) 877 : { 878 13987439 : for (j = i + 1; j < n_wc; j++) 879 13901656 : wc_buffer[j - 1] = wc_buffer[j]; 880 85783 : n_wc--; 881 85783 : i--; 882 : 883 85783 : if (i == last_start) 884 82904 : last_cc = 0; 885 : else 886 2879 : last_cc = COMBINING_CLASS (wc_buffer[i - 1]); 887 : 888 85783 : continue; 889 : } 890 : 891 1459630 : if (cc == 0) 892 1426841 : last_start = i; 893 : 894 1459630 : last_cc = cc; 895 : } 896 : } 897 : 898 54130 : wc_buffer[n_wc] = 0; 899 : 900 54130 : return wc_buffer; 901 : } 902 : 903 : /* 904 : * g_utf8_normalize: 905 : * @str: a UTF-8 encoded string. 906 : * @len: length of @str, in bytes, or -1 if @str is nul-terminated. 907 : * @mode: the type of normalization to perform. 908 : * 909 : * Converts a string into canonical form, standardizing 910 : * such issues as whether a character with an accent 911 : * is represented as a base character and combining 912 : * accent or as a single precomposed character. The 913 : * string has to be valid UTF-8, otherwise %NULL is 914 : * returned. You should generally call g_utf8_normalize() 915 : * before comparing two Unicode strings. 916 : * 917 : * The normalization mode %G_NORMALIZE_DEFAULT only 918 : * standardizes differences that do not affect the 919 : * text content, such as the above-mentioned accent 920 : * representation. %G_NORMALIZE_ALL also standardizes 921 : * the "compatibility" characters in Unicode, such 922 : * as SUPERSCRIPT THREE to the standard forms 923 : * (in this case DIGIT THREE). Formatting information 924 : * may be lost but for most text operations such 925 : * characters should be considered the same. 926 : * 927 : * %G_NORMALIZE_DEFAULT_COMPOSE and %G_NORMALIZE_ALL_COMPOSE 928 : * are like %G_NORMALIZE_DEFAULT and %G_NORMALIZE_ALL, 929 : * but returned a result with composed forms rather 930 : * than a maximally decomposed form. This is often 931 : * useful if you intend to convert the string to 932 : * a legacy encoding or pass it to a system with 933 : * less capable Unicode handling. 934 : * 935 : * Return value: a newly allocated string, that is the 936 : * normalized form of @str, or %NULL if @str is not 937 : * valid UTF-8. 938 : **/ 939 : static gchar * 940 189 : g_utf8_normalize (const gchar * str, gssize len, GNormalizeMode mode) 941 : { 942 189 : gunichar *result_wc = _g_utf8_normalize_wc (str, len, mode); 943 189 : gchar *result = NULL; 944 : 945 189 : if (result_wc) 946 189 : result = g_ucs4_to_utf8 (result_wc, -1, NULL, NULL); 947 : 948 189 : g_free (result_wc); 949 : 950 189 : return result; 951 : } 952 : 953 : /* Public Libidn API starts here. */ 954 : 955 : /** 956 : * stringprep_utf8_to_unichar: 957 : * @p: a pointer to Unicode character encoded as UTF-8 958 : * 959 : * Converts a sequence of bytes encoded as UTF-8 to a Unicode character. 960 : * If @p does not point to a valid UTF-8 encoded character, results are 961 : * undefined. 962 : * 963 : * Return value: the resulting character. 964 : **/ 965 : uint32_t 966 364 : stringprep_utf8_to_unichar (const char *p) 967 : { 968 364 : return g_utf8_get_char (p); 969 : } 970 : 971 : /** 972 : * stringprep_unichar_to_utf8: 973 : * @c: a ISO10646 character code 974 : * @outbuf: output buffer, must have at least 6 bytes of space. 975 : * If %NULL, the length will be computed and returned 976 : * and nothing will be written to @outbuf. 977 : * 978 : * Converts a single character to UTF-8. 979 : * 980 : * Return value: number of bytes written. 981 : **/ 982 : int 983 364 : stringprep_unichar_to_utf8 (uint32_t c, char *outbuf) 984 : { 985 364 : return g_unichar_to_utf8 (c, outbuf); 986 : } 987 : 988 : #include <unistr.h> 989 : 990 : /** 991 : * stringprep_utf8_to_ucs4: 992 : * @str: a UTF-8 encoded string 993 : * @len: the maximum length of @str to use. If @len < 0, then 994 : * the string is nul-terminated. 995 : * @items_written: location to store the number of characters in the 996 : * result, or %NULL. 997 : * 998 : * Convert a string from UTF-8 to a 32-bit fixed width representation 999 : * as UCS-4. The function now performs error checking to verify that 1000 : * the input is valid UTF-8 (before it was documented to not do error 1001 : * checking). 1002 : * 1003 : * Return value: a pointer to a newly allocated UCS-4 string. 1004 : * This value must be deallocated by the caller. 1005 : **/ 1006 : uint32_t * 1007 75525 : stringprep_utf8_to_ucs4 (const char *str, ssize_t len, size_t *items_written) 1008 : { 1009 : size_t n; 1010 : 1011 75525 : if (len < 0) 1012 75525 : n = strlen (str); 1013 : else 1014 0 : n = len; 1015 : 1016 75525 : if (u8_check ((const uint8_t *) str, n)) 1017 1711 : return NULL; 1018 : 1019 73814 : return g_utf8_to_ucs4_fast (str, (glong) len, (glong *) items_written); 1020 : } 1021 : 1022 : /** 1023 : * stringprep_ucs4_to_utf8: 1024 : * @str: a UCS-4 encoded string 1025 : * @len: the maximum length of @str to use. If @len < 0, then 1026 : * the string is terminated with a 0 character. 1027 : * @items_read: location to store number of characters read read, or %NULL. 1028 : * @items_written: location to store number of bytes written or %NULL. 1029 : * The value here stored does not include the trailing 0 1030 : * byte. 1031 : * 1032 : * Convert a string from a 32-bit fixed width representation as UCS-4. 1033 : * to UTF-8. The result will be terminated with a 0 byte. 1034 : * 1035 : * Return value: a pointer to a newly allocated UTF-8 string. 1036 : * This value must be deallocated by the caller. 1037 : * If an error occurs, %NULL will be returned. 1038 : **/ 1039 : char * 1040 138175 : stringprep_ucs4_to_utf8 (const uint32_t * str, ssize_t len, 1041 : size_t *items_read, size_t *items_written) 1042 : { 1043 138175 : return g_ucs4_to_utf8 (str, len, (glong *) items_read, 1044 : (glong *) items_written); 1045 : } 1046 : 1047 : /** 1048 : * stringprep_utf8_nfkc_normalize: 1049 : * @str: a UTF-8 encoded string. 1050 : * @len: length of @str, in bytes, or -1 if @str is nul-terminated. 1051 : * 1052 : * Converts a string into canonical form, standardizing 1053 : * such issues as whether a character with an accent 1054 : * is represented as a base character and combining 1055 : * accent or as a single precomposed character. 1056 : * 1057 : * The normalization mode is NFKC (ALL COMPOSE). It standardizes 1058 : * differences that do not affect the text content, such as the 1059 : * above-mentioned accent representation. It standardizes the 1060 : * "compatibility" characters in Unicode, such as SUPERSCRIPT THREE to 1061 : * the standard forms (in this case DIGIT THREE). Formatting 1062 : * information may be lost but for most text operations such 1063 : * characters should be considered the same. It returns a result with 1064 : * composed forms rather than a maximally decomposed form. 1065 : * 1066 : * Return value: a newly allocated string, that is the 1067 : * NFKC normalized form of @str. 1068 : **/ 1069 : char * 1070 369 : stringprep_utf8_nfkc_normalize (const char *str, ssize_t len) 1071 : { 1072 : size_t n; 1073 : 1074 369 : if (len < 0) 1075 1 : n = strlen (str); 1076 : else 1077 368 : n = len; 1078 : 1079 369 : if (u8_check ((const uint8_t *) str, n)) 1080 180 : return NULL; 1081 : 1082 189 : return g_utf8_normalize (str, len, G_NORMALIZE_NFKC); 1083 : } 1084 : 1085 : #include <stdio.h> 1086 : /** 1087 : * stringprep_ucs4_nfkc_normalize: 1088 : * @str: a Unicode string. 1089 : * @len: length of @str array, or -1 if @str is nul-terminated. 1090 : * 1091 : * Converts a UCS4 string into canonical form, see 1092 : * stringprep_utf8_nfkc_normalize() for more information. 1093 : * 1094 : * Return value: a newly allocated Unicode string, that is the NFKC 1095 : * normalized form of @str. 1096 : **/ 1097 : uint32_t * 1098 54021 : stringprep_ucs4_nfkc_normalize (const uint32_t * str, ssize_t len) 1099 : { 1100 : char *p; 1101 : uint32_t *result_wc; 1102 : 1103 54021 : p = stringprep_ucs4_to_utf8 (str, len, 0, 0); 1104 54021 : if (!p) 1105 80 : return NULL; 1106 : 1107 53941 : result_wc = _g_utf8_normalize_wc (p, -1, G_NORMALIZE_NFKC); 1108 53941 : free (p); 1109 : 1110 53941 : return result_wc; 1111 : }