dlib C++ Library - unicode.h

// Copyright (C) 2007 Davis E. King (davis@dlib.net), and Nils Labugt
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_UNICODe_H_
#define DLIB_UNICODe_H_
#include "../uintn.h"
#include "../algs.h"
#include "unicode_abstract.h"
#include <string>
#include <cstring>
#include <fstream>
namespace dlib
{
// ----------------------------------------------------------------------------------------
 using unichar = char32_t;
 using ustring = std::basic_string<unichar>;
// ----------------------------------------------------------------------------------------
 namespace unicode_helpers
 {
 template <
 typename charT,
 typename forward_iterator
 >
 int u8_to_u32(
 charT& result,
 forward_iterator ibegin,
 forward_iterator iend
 )
 /*!
 requires
 - ibegin == iterator pointing to the start of the range
 - iend == iterator pointing to the end of the range
 ensures
 - if (there just wasn't any more data and ibegin >= iend) then
 - returns 0
 - else if (we decoded another character without error) then
 - #result == the decoded character
 - returns the number of bytes consumed to make this character
 - else
 - some error occurred
 - returns -1
 !*/
 {
 if (ibegin >= iend)
 return 0;
 int val = static_cast<unsigned char>(*ibegin);
 unichar ch[4];
 ch[0] = zero_extend_cast<unichar>(val);
 if (ch[0] < 0x80)
 {
 result = static_cast<charT>(ch[0]);
 return 1;
 }
 if ((ch[0] & ~0x3F ) == 0x80)
 {
 // invalid leading byte
 return -1;
 }
 if ((ch[0] & ~0x1F) == 0xC0)
 {
 if (++ibegin == iend)
 return -1;
 val = static_cast<unsigned char>(*ibegin);
 ch[1] = zero_extend_cast<unichar>(val);
 if ((ch[1] & ~0x3F ) != 0x80)
 return -1; // invalid tail
 if ((ch[0] & ~0x01 ) == 0xC0)
 return -1; // overlong form
 ch[0] &= 0x1F;
 ch[1] &= 0x3F;
 result = static_cast<charT>((ch[0] << 6) | ch[1]);
 return 2;
 }
 if ((ch[0] & ~0x0F ) == 0xE0)
 {
 for (unsigned n = 1; n < 3; ++n)
 {
 if (++ibegin == iend)
 return -1;
 val = static_cast<unsigned char>(*ibegin);
 ch[n] = zero_extend_cast<unichar>(val);
 if ((ch[n] & ~0x3F) != 0x80)
 return -1; // invalid tail
 ch[n] &= 0x3F;
 }
 ch[0] &= 0x0F;
 result = static_cast<charT>((ch[0] << 12) | (ch[1] << 6) | ch[2]);
 if (result < 0x0800)
 return -1; // overlong form
 if (result >= 0xD800 && result < 0xE000)
 return -1; // invalid character (UTF-16 surrogate pairs)
 if (result >= 0xFDD0 && result <= 0xFDEF)
 return -1; // noncharacter
 if (result >= 0xFFFE)
 return -1; // noncharacter
 return 3;
 }
 if ((ch[0] & ~0x07) == 0xF0)
 {
 for (unsigned n = 1; n < 4; ++n)
 {
 if (++ibegin == iend)
 return -1;
 val = static_cast<unsigned char>(*ibegin);
 ch[n] = zero_extend_cast<unichar>(val);
 if ((ch[n] & ~0x3F) != 0x80)
 return -1; // invalid tail
 ch[n] &= 0x3F;
 }
 if ((ch[0] ^ 0xF6) < 4)
 return -1;
 ch[0] &= 0x07;
 result = static_cast<charT>((ch[0] << 18) | (ch[1] << 12) | (ch[2] << 6) | ch[3]);
 if (result < 0x10000)
 return -1; // overlong form
 if ((result & 0xFFFF) >= 0xFFFE)
 return -1; // noncharacter
 return 4;
 }
 return -1;
 }
 // ------------------------------------------------------------------------------------
 template <typename charT>
 class basic_utf8_streambuf : public std::basic_streambuf<charT>
 {
 public:
 basic_utf8_streambuf (
 std::ifstream& fin_
 ) :
 fin(fin_)
 {
 this->setg(in_buffer+max_putback, 
 in_buffer+max_putback, 
 in_buffer+max_putback);
 }
 protected:
 using int_type = typename std::basic_streambuf<charT>::int_type;
 // input functions
 int_type underflow( 
 )
 {
 if (this->gptr() < this->egptr())
 {
 return zero_extend_cast<int_type>(*this->gptr());
 }
 int num_put_back = static_cast<int>(this->gptr() - this->eback());
 if (num_put_back > max_putback)
 {
 num_put_back = max_putback;
 }
 // copy the putback characters into the putback end of the in_buffer
 std::memmove(in_buffer+(max_putback-num_put_back), this->gptr()-num_put_back, num_put_back);
 // fill the buffer with characters
 int n = in_buffer_size-max_putback;
 int i;
 for (i = 0; i < n; ++i)
 {
 charT ch;
 using iter_type = std::istreambuf_iterator<char>;
 if (unicode_helpers::u8_to_u32(ch, iter_type(fin), iter_type()) > 0)
 {
 (in_buffer+max_putback)[i] = ch;
 }
 else
 {
 break;
 }
 }
 if (i == 0)
 {
 // an error occurred or we hit EOF
 return EOF;
 }
 // reset in_buffer pointers
 this->setg (in_buffer+(max_putback-num_put_back),
 in_buffer+max_putback,
 in_buffer+max_putback+i);
 return zero_extend_cast<int_type>(*this->gptr());
 }
 private:
 std::ifstream& fin;
 static const int max_putback = 4;
 static const int in_buffer_size = 10;
 charT in_buffer[in_buffer_size];
 };
 static const unichar SURROGATE_FIRST_TOP = 0xD800;
 static const unichar SURROGATE_SECOND_TOP = 0xDC00;
 static const unichar SURROGATE_CLEARING_MASK = 0x03FF;
 static const unichar SURROGATE_TOP = SURROGATE_FIRST_TOP;
 static const unichar SURROGATE_END = 0xE000;
 static const unichar SMP_TOP = 0x10000;
 static const int VALID_BITS = 10;
 }
// ----------------------------------------------------------------------------------------
 template <typename T>
 bool is_combining_char(
 const T ch_
 )
 {
 const unichar ch = zero_extend_cast<unichar>(ch_);
 if (ch < 0x300)
 return false;
 if (ch < 0x370)
 return true;
 if (ch < 0x800) {
 if (ch < 0x483)
 return false;
 if (ch < 0x48A)
 return true;
 if (ch < 0x591)
 return false;
 if (ch < 0x5D0) {
 if (ch == 0x5C0)
 return false;
 if (ch == 0x5C3)
 return false;
 if (ch == 0x5C6)
 return false;
 return true;
 }
 if (ch < 0x610)
 return false;
 if (ch < 0x616)
 return true;
 if (ch < 0x64B)
 return false;
 if (ch < 0x660)
 return true;
 if (ch == 0x670)
 return true;
 if (ch < 0x6D6)
 return false;
 if (ch < 0x6EE) {
 if (ch == 0x6DD)
 return false;
 if (ch == 0x6E5)
 return false;
 if (ch == 0x6E6)
 return false;
 if (ch == 0x6E9)
 return false;
 return true;
 }
 if (ch == 0x711)
 return true;
 if (ch < 0x730)
 return false;
 if (ch < 0x74B)
 return true;
 if (ch < 0x7A6)
 return false;
 if (ch < 0x7B1)
 return true;
 if (ch < 0x7EB)
 return false;
 if (ch < 0x7F4)
 return true;
 return false;
 }
 if (ch < 0xA00) {
 if (ch < 0x901)
 return false;
 if (ch < 0x904)
 return true;
 if (ch < 0x93C)
 return false;
 if (ch < 0x955) {
 if (ch == 0x93D)
 return false;
 if (ch == 0x950)
 return false;
 return true;
 }
 if (ch < 0x962)
 return false;
 if (ch < 0x964)
 return true;
 if (ch < 0x981)
 return false;
 if (ch < 0x984)
 return true;
 if (ch < 0x9BC)
 return false;
 if (ch < 0x9D8) {
 if (ch == 0x9BD)
 return false;
 if (ch == 0x9CE)
 return false;
 return true;
 }
 if (ch < 0x9E2)
 return false;
 if (ch < 0x9E4)
 return true;
 return false;
 }
 if (ch < 0xC00) {
 if (ch < 0xA01)
 return false;
 if (ch < 0xA04)
 return true;
 if (ch < 0xA3C)
 return false;
 if (ch < 0xA4E)
 return true;
 if (ch < 0xA70)
 return false;
 if (ch < 0xA72)
 return true;
 if (ch < 0xA81)
 return false;
 if (ch < 0xA84)
 return true;
 if (ch < 0xABC)
 return false;
 if (ch < 0xACE) {
 if (ch == 0xABD)
 return false;
 return true;
 }
 if (ch < 0xAE2)
 return false;
 if (ch < 0xAE4)
 return true;
 if (ch < 0xB01)
 return false;
 if (ch < 0xB04)
 return true;
 if (ch < 0xB3C)
 return false;
 if (ch < 0xB58) {
 if (ch == 0xB3D)
 return false;
 return true;
 }
 if (ch == 0xB82)
 return true;
 if (ch < 0xBBE)
 return false;
 if (ch < 0xBD8)
 return true;
 if (ch == 0xBF4)
 return true;
 if (ch == 0xBF8)
 return true;
 return false;
 }
 if (ch < 0xE00) {
 if (ch < 0xC01)
 return false;
 if (ch < 0xC04)
 return true;
 if (ch < 0xC3E)
 return false;
 if (ch < 0xC57)
 return true;
 if (ch < 0xC82)
 return false;
 if (ch < 0xC84)
 return true;
 if (ch < 0xCBC)
 return false;
 if (ch < 0xCD7) {
 if (ch == 0xCBD)
 return false;
 return true;
 }
 if (ch < 0xCE2)
 return false;
 if (ch < 0xCE4)
 return true;
 if (ch < 0xD02)
 return false;
 if (ch < 0xD04)
 return true;
 if (ch < 0xD3E)
 return false;
 if (ch < 0xD58)
 return true;
 if (ch < 0xD82)
 return false;
 if (ch < 0xD84)
 return true;
 if (ch < 0xDCA)
 return false;
 if (ch < 0xDF4)
 return true;
 return false;
 }
 if (ch < 0x1000) {
 if (ch == 0xE31)
 return true;
 if (ch < 0xE34)
 return false;
 if (ch < 0xE3B)
 return true;
 if (ch < 0xE47)
 return false;
 if (ch < 0xE4F)
 return true;
 if (ch == 0xEB1)
 return true;
 if (ch < 0xEB4)
 return false;
 if (ch < 0xEBD)
 return true;
 if (ch < 0xEC8)
 return false;
 if (ch < 0xECE)
 return true;
 if (ch < 0xF18)
 return false;
 if (ch < 0xF1A)
 return true;
 if (ch == 0xF35)
 return true;
 if (ch == 0xF37)
 return true;
 if (ch == 0xF39)
 return true;
 if (ch < 0xF3E)
 return false;
 if (ch < 0xF40)
 return true;
 if (ch < 0xF71)
 return false;
 if (ch < 0xF88) {
 if (ch == 0xF85)
 return false;
 return true;
 }
 if (ch < 0xF90)
 return false;
 if (ch < 0xFBD)
 return true;
 if (ch == 0xFC6)
 return true;
 return false;
 }
 if (ch < 0x1800) {
 if (ch < 0x102C)
 return false;
 if (ch < 0x1040)
 return true;
 if (ch < 0x1056)
 return false;
 if (ch < 0x105A)
 return true;
 if (ch == 0x135F)
 return true;
 if (ch < 0x1712)
 return false;
 if (ch < 0x1715)
 return true;
 if (ch < 0x1732)
 return false;
 if (ch < 0x1735)
 return true;
 if (ch < 0x1752)
 return false;
 if (ch < 0x1754)
 return true;
 if (ch < 0x1772)
 return false;
 if (ch < 0x1774)
 return true;
 if (ch < 0x17B6)
 return false;
 if (ch < 0x17D4)
 return true;
 if (ch == 0x17DD)
 return true;
 return false;
 }
 if (ch < 0x2000) {
 if (ch < 0x180B)
 return false;
 if (ch < 0x180E)
 return true;
 if (ch == 0x18A9)
 return true;
 if (ch < 0x1920)
 return false;
 if (ch < 0x193C)
 return true;
 if (ch < 0x19B0)
 return false;
 if (ch < 0x19C1)
 return true;
 if (ch < 0x19C8)
 return false;
 if (ch < 0x19CA)
 return true;
 if (ch < 0x1A17)
 return false;
 if (ch < 0x1A1C)
 return true;
 if (ch < 0x1B00)
 return false;
 if (ch < 0x1B05)
 return true;
 if (ch < 0x1B34)
 return false;
 if (ch < 0x1B45)
 return true;
 if (ch < 0x1B6B)
 return false;
 if (ch < 0x1B74)
 return true;
 if (ch < 0x1DC0)
 return false;
 if (ch < 0x1E00)
 return true;
 return false;
 }
 if (ch < 0x20D0)
 return false;
 if (ch < 0x2100)
 return true;
 if (ch < 0x302A)
 return false;
 if (ch < 0x3030)
 return true;
 if (ch < 0x3099)
 return false;
 if (ch < 0x309B)
 return true;
 if (ch == 0xA802)
 return true;
 if (ch == 0xA806)
 return true;
 if (ch == 0xA80B)
 return true;
 if (ch < 0xA823)
 return false;
 if (ch < 0xA828)
 return true;
 if (ch == 0xFB1E)
 return true;
 if (ch < 0xFE00)
 return false;
 if (ch < 0xFE10)
 return true;
 if (ch < 0xFE20)
 return false;
 if (ch < 0xFE30)
 return true;
 if (ch < 0x10A01)
 return false;
 if (ch < 0x10A10)
 return true;
 if (ch < 0x10A38)
 return false;
 if (ch < 0x10A40)
 return true;
 if (ch < 0x1D165)
 return false;
 if (ch < 0x1D16A)
 return true;
 if (ch < 0x1D16D)
 return false;
 if (ch < 0x1D173)
 return true;
 if (ch < 0x1D17B)
 return false;
 if (ch < 0x1D183)
 return true;
 if (ch < 0x1D185)
 return false;
 if (ch < 0x1D18C)
 return true;
 if (ch < 0x1D1AA)
 return false;
 if (ch < 0x1D1AE)
 return true;
 if (ch < 0x1D242)
 return false;
 if (ch < 0x1D245)
 return true;
 if (ch < 0xE0100)
 return false;
 if (ch < 0xE01F0)
 return true;
 return false;
 }
// ----------------------------------------------------------------------------------------
 void unichar_to_surrogate_pair(unichar input, unichar &first, unichar &second);
// ----------------------------------------------------------------------------------------
 template <typename T> bool is_surrogate(T ch)
 {
 using namespace unicode_helpers;
 return (zero_extend_cast<unichar>(ch) >= SURROGATE_TOP && 
 zero_extend_cast<unichar>(ch) < SURROGATE_END);
 }
// ----------------------------------------------------------------------------------------
 template <typename T> unichar surrogate_pair_to_unichar(T first, T second)
 {
 using namespace unicode_helpers;
 return ((first & SURROGATE_CLEARING_MASK) << VALID_BITS) | ((second & SURROGATE_CLEARING_MASK) + SMP_TOP);
 }
 //110110 0000000000
 //110111 0000000000
// ----------------------------------------------------------------------------------------
 class invalid_utf8_error : public error
 {
 public:
 invalid_utf8_error():error(EUTF8_TO_UTF32) {}
 };
 template <typename forward_iterator, typename unary_op>
 inline void convert_to_utf32(
 forward_iterator ibegin,
 forward_iterator iend,
 unary_op op
 )
 {
 using char_type = std::decay_t<decltype(*ibegin)>;
 static_assert(std::is_same<char_type, char>::value ||
 std::is_same<char_type, wchar_t>::value ||
 std::is_same<char_type, unichar>::value,
 "char_type must be either char or unichar");
 if (std::is_same<char_type, unichar>::value)
 {
 while (ibegin != iend)
 op(*(ibegin++));
 return;
 }
 if (std::is_same<char_type, wchar_t>::value)
 {
 // Unix
 if (sizeof(wchar_t) == 4)
 {
 while (ibegin != iend)
 op(static_cast<unichar>(*(ibegin++)));
 return;
 }
 // Win32
 if (sizeof(wchar_t) == 2)
 {
 while (ibegin != iend)
 {
 if (is_surrogate(*ibegin))
 {
 op(surrogate_pair_to_unichar(*ibegin, *(ibegin+ 1)));
 ibegin += 2;
 }
 else
 {
 op(zero_extend_cast<unichar>(*ibegin));
 ibegin += 1;
 }
 }
 return;
 }
 throw invalid_utf8_error();
 }
 if (std::is_same<char_type, char>::value)
 {
 unichar ch;
 int status = 0;
 while (ibegin != iend)
 {
 status = unicode_helpers::u8_to_u32(ch, ibegin, iend);
 if (status > 0)
 {
 op(ch);
 ibegin += status;
 }
 else
 {
 break;
 }
 }
 if (status < 0)
 throw invalid_utf8_error();
 }
 }
 template <typename char_type, typename traits, typename alloc>
 const ustring convert_to_utf32 (
 const std::basic_string<char_type, traits, alloc>& str
 )
 {
 ustring temp;
 temp.reserve(str.size());
 convert_to_utf32(str.begin(), str.end(), [&](unichar ch) { temp.push_back(ch); });
 return temp;
 }
 const ustring convert_utf8_to_utf32(const std::string& str);
 const ustring convert_wstring_to_utf32(const std::wstring& str);
// ----------------------------------------------------------------------------------------
 const std::wstring convert_utf32_to_wstring (
 const ustring &src
 );
 const std::wstring convert_mbstring_to_wstring (
 const std::string &src
 );
 const std::string convert_wstring_to_mbstring(
 const std::wstring &src
 );
// ----------------------------------------------------------------------------------------
 template <typename charT>
 class basic_utf8_ifstream : public std::basic_istream<charT>
 {
 public:
 basic_utf8_ifstream (
 ) : std::basic_istream<charT>(&buf), buf(fin) {}
 basic_utf8_ifstream (
 const char* file_name,
 std::ios_base::openmode mode = std::ios::in 
 ) : 
 std::basic_istream<charT>(&buf),
 buf(fin)
 {
 fin.open(file_name,mode);
 // make this have the same error state as fin
 this->clear(fin.rdstate());
 }
 basic_utf8_ifstream (
 const std::string& file_name,
 std::ios_base::openmode mode = std::ios::in 
 ) : 
 std::basic_istream<charT>(&buf),
 buf(fin)
 {
 fin.open(file_name.c_str(),mode);
 // make this have the same error state as fin
 this->clear(fin.rdstate());
 }
 void open(
 const std::string& file_name,
 std::ios_base::openmode mode = std::ios::in 
 )
 {
 open(file_name.c_str(),mode);
 }
 void open (
 const char* file_name,
 std::ios_base::openmode mode = std::ios::in 
 )
 {
 fin.close();
 fin.clear();
 fin.open(file_name,mode);
 // make this have the same error state as fin
 this->clear(fin.rdstate());
 }
 void close (
 )
 {
 fin.close();
 // make this have the same error state as fin
 this->clear(fin.rdstate());
 }
 private:
 std::ifstream fin;
 unicode_helpers::basic_utf8_streambuf<charT> buf;
 };
 using utf8_uifstream = basic_utf8_ifstream<unichar>;
 using utf8_wifstream = basic_utf8_ifstream<wchar_t>;
// ----------------------------------------------------------------------------------------
}
#ifdef NO_MAKEFILE
#include "unicode.cpp"
#endif
#endif // DLIB_UNICODe_H_