upstream/wesnoth
1
0
Fork 0
mirror of https://github.com/wesnoth/wesnoth synced 2025-04-30 17:55:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Follow up the utf8 namespace with ucs4 and utf16 namespaces

Browse Source
This commit is contained in:
Alexander van Gessel 2014-03-23 04:24:02 +01:00
parent a25a0b7d35
commit 6b5f357b6e
18 changed files with 90 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/construct_dialog.cpp
View File

@ -910,7 +910,7 @@ void filter_textbox::delete_item(int selection) {
/* dialog_.set_menu_items(filtered_items_); */
}
void filter_textbox::handle_text_changed(const ucs4_string& text) {
void filter_textbox::handle_text_changed(const ucs4::string& text) {
const std::vector<std::string> words = utils::split(utils::ucs4string_to_string(text),' ');
if (words == last_words)
return;

2
src/construct_dialog.hpp
View File

@ -145,7 +145,7 @@ private:
std::vector<std::string> last_words;
size_t header_row_;
gui::dialog& dialog_;
virtual void handle_text_changed(const ucs4_string& text);
virtual void handle_text_changed(const ucs4::string& text);
};
class dialog_button : public button {

4
src/desktop_util.cpp
View File

@ -91,8 +91,8 @@ bool open_object(const std::string& path_or_url)
LOG_DU << "open_object(): on Win32, will use ShellExecute()\n";
ucs4_string u4path = utils::string_to_ucs4string(path_or_url);
utf16_string u16path = utils::ucs4string_to_utf16string(u4path);
ucs4::string u4path = utils::string_to_ucs4string(path_or_url);
utf16::string u16path = utils::ucs4string_to_utf16string(u4path);
u16path.push_back(wchar_t(0)); // Make wpath NULL-terminated
const ptrdiff_t res = reinterpret_cast<ptrdiff_t>(ShellExecute(NULL, L"open", &u16path.front(), NULL, NULL, SW_SHOW));

2
src/help.cpp
View File

@ -3345,7 +3345,7 @@ std::string get_first_word(const std::string &s)
if (ch == utf8::iterator::end(re))
return re;
ucs4char firstchar = *ch;
ucs4::char_t firstchar = *ch;
if (font::is_cjk_char(firstchar)) {
re = utils::ucs4char_to_string(firstchar);
}

2
src/hotkey/hotkey_item.cpp
View File

@ -360,7 +360,7 @@ void hotkey_item::load_from_config(const config& cfg)
return;
}
ucs4_string wkey = utils::string_to_ucs4string(key);
ucs4::string wkey = utils::string_to_ucs4string(key);
// They may really want a specific key on the keyboard:
// we assume that any single character keyname is a character.

14
src/marked-up_text.cpp
View File

@ -261,7 +261,7 @@ bool is_format_char(char c)
}
}
bool is_cjk_char(const ucs4char ch)
bool is_cjk_char(const ucs4::char_t ch)
{
/**
* You can check these range at http://unicode.org/charts/
@ -356,7 +356,7 @@ namespace {
* CJK 标点符号 (CJK punctuations)
* http://www.unicode.org/charts/PDF/U3000.pdf
*/
inline bool no_break_after(const ucs4char ch)
inline bool no_break_after(const ucs4::char_t ch)
{
return
/**
@ -377,7 +377,7 @@ inline bool no_break_after(const ucs4char ch)
ch == 0x3016 || ch == 0x301a || ch == 0x301d;
}
inline bool no_break_before(const ucs4char ch)
inline bool no_break_before(const ucs4::char_t ch)
{
return
/**
@ -414,7 +414,7 @@ inline bool no_break_before(const ucs4char ch)
ch == 0x301b || ch == 0x301e;
}
inline bool break_before(const ucs4char ch)
inline bool break_before(const ucs4::char_t ch)
{
if(no_break_before(ch))
return false;
@ -422,7 +422,7 @@ inline bool break_before(const ucs4char ch)
return is_cjk_char(ch);
}
inline bool break_after(const ucs4char ch)
inline bool break_after(const ucs4::char_t ch)
{
if(no_break_after(ch))
return false;
@ -456,7 +456,7 @@ std::string word_wrap_text(const std::string& unwrapped_text, int font_size,
if(start_of_line) {
line_width = 0;
format_string.clear();
while(ch != end && *ch < static_cast<ucs4char>(0x100)
while(ch != end && *ch < static_cast<ucs4::char_t>(0x100)
&& is_format_char(*ch) && !ch.next_is_end()) {
format_string.append(ch.substr().first, ch.substr().second);
@ -479,7 +479,7 @@ std::string word_wrap_text(const std::string& unwrapped_text, int font_size,
current_word = *ch;
++ch;
} else {
ucs4char previous = 0;
ucs4::char_t previous = 0;
for(;ch != utf8::iterator::end(unwrapped_text) &&
*ch != ' ' && *ch != '\n'; ++ch) {

4
src/marked-up_text.hpp
View File

@ -96,12 +96,12 @@ std::string del_tags(const std::string& text);
bool is_format_char(char c);
/**
* Determine if a ucs4char is a CJK character
* Determine if a ucs4::char_t is a CJK character
*
* @retval true Input-char is a CJK char
* @retval false Input-char is a not CJK char.
*/
bool is_cjk_char(const ucs4char ch);
bool is_cjk_char(const ucs4::char_t ch);
/** Create string of color-markup, such as "<255,255,0>" for yellow. */
std::string color2markup(const SDL_Color &color);

20
src/race.cpp
View File

@ -44,12 +44,12 @@ static const config &empty_topics() {
return cfg;
}
static void add_prefixes(const ucs4_string& str, size_t length, markov_prefix_map& res)
static void add_prefixes(const ucs4::string& str, size_t length, markov_prefix_map& res)
{
for(size_t i = 0; i <= str.size(); ++i) {
const size_t start = i > length ? i - length : 0;
const ucs4_string key(str.begin() + start, str.begin() + i);
const ucs4char c = i != str.size() ? str[i] : 0;
const ucs4::string key(str.begin() + start, str.begin() + i);
const ucs4::char_t c = i != str.size() ? str[i] : 0;
res[key].push_back(c);
}
}
@ -65,13 +65,13 @@ static markov_prefix_map markov_prefixes(const std::vector<std::string>& items,
return res;
}
static ucs4_string markov_generate_name(const markov_prefix_map& prefixes,
static ucs4::string markov_generate_name(const markov_prefix_map& prefixes,
size_t chain_size, size_t max_len, rand_rng::simple_rng* rng)
{
if(chain_size == 0)
return ucs4_string();
return ucs4::string();
ucs4_string prefix, res;
ucs4::string prefix, res;
// Since this function is called in the name description in a MP game it
// uses the local locale. The locale between players can be different and
@ -96,7 +96,7 @@ static ucs4_string markov_generate_name(const markov_prefix_map& prefixes,
return res;
}
const ucs4char c = i->second[random[j++]%i->second.size()];
const ucs4::char_t c = i->second[random[j++]%i->second.size()];
if(c == 0) {
return res;
}
@ -119,17 +119,17 @@ static ucs4_string markov_generate_name(const markov_prefix_map& prefixes,
// name has end-of-string as a possible next character in the
// markov prefix map. If no valid ending is found, use the
// originally generated name.
ucs4_string originalRes = res;
ucs4::string originalRes = res;
int prefixLen;
while(!res.empty()) {
prefixLen = chain_size < res.size() ? chain_size : res.size();
prefix = ucs4_string(res.end() - prefixLen, res.end());
prefix = ucs4::string(res.end() - prefixLen, res.end());
const markov_prefix_map::const_iterator i = prefixes.find(prefix);
if (i == prefixes.end() || i->second.empty()) {
return res;
}
if (std::find(i->second.begin(), i->second.end(), static_cast<ucs4char>(0))
if (std::find(i->second.begin(), i->second.end(), static_cast<ucs4::char_t>(0))
!= i->second.end()) {
// This ending is valid.
return res;

2
src/race.hpp
View File

@ -23,7 +23,7 @@ namespace rand_rng {
} // namespace rand_rng
typedef std::map<ucs4_string, ucs4_string > markov_prefix_map;
typedef std::map<ucs4::string, ucs4::string > markov_prefix_map;
class unit_race
{

36
src/serialization/string_utils.cpp
View File

@ -35,7 +35,7 @@ static lg::log_domain log_engine("engine");
#define ERR_NG LOG_STREAM(err, log_engine)
namespace {
size_t byte_size_from_ucs4_codepoint(ucs4char ch)
size_t byte_size_from_ucs4_codepoint(ucs4::char_t ch)
{
if(ch < (1u << 7))
return 1;
@ -862,14 +862,14 @@ std::vector< std::pair< int, int > > parse_ranges(std::string const &str)
return to_return;
}
std::string ucs4string_to_string(const ucs4_string &src)
std::string ucs4string_to_string(const ucs4::string &src)
{
std::string ret;
try {
for(ucs4_string::const_iterator i = src.begin(); i != src.end(); ++i) {
for(ucs4::string::const_iterator i = src.begin(); i != src.end(); ++i) {
unsigned int count;
ucs4char ch = *i;
ucs4::char_t ch = *i;
// Determine the bytes required
count = byte_size_from_ucs4_codepoint(ch);
@ -897,16 +897,16 @@ std::string ucs4string_to_string(const ucs4_string &src)
}
}
std::string ucs4char_to_string(const ucs4char c)
std::string ucs4char_to_string(const ucs4::char_t c)
{
ucs4_string s;
ucs4::string s;
s.push_back(c);
return ucs4string_to_string(s);
}
ucs4_string string_to_ucs4string(const std::string &src)
ucs4::string string_to_ucs4string(const std::string &src)
{
ucs4_string res;
ucs4::string res;
try {
utf8::iterator i1(src);
@ -926,18 +926,18 @@ ucs4_string string_to_ucs4string(const std::string &src)
return res;
}
utf16_string ucs4string_to_utf16string(const ucs4_string &src)
utf16::string ucs4string_to_utf16string(const ucs4::string &src)
{
utf16_string res;
utf16::string res;
const Uint32 bit17 = 0x10000;
BOOST_FOREACH(const ucs4char &u4, src) {
BOOST_FOREACH(const ucs4::char_t &u4, src) {
if(u4 < bit17)
res.push_back(static_cast<wchar_t>(u4));
else {
const Uint32 char20 = u4 - bit17;
assert(char20 < (1 << 20));
const ucs4char lead = 0xD800 + (char20 >> 10);
const ucs4char trail = 0xDC00 + (char20 & 0x3FF);
const ucs4::char_t lead = 0xD800 + (char20 >> 10);
const ucs4::char_t trail = 0xDC00 + (char20 & 0x3FF);
assert(lead < bit17);
assert(trail < bit17);
res.push_back(static_cast<wchar_t>(lead));
@ -950,7 +950,7 @@ utf16_string ucs4string_to_utf16string(const ucs4_string &src)
void truncate_as_ucs4string(std::string& str, const size_t size)
{
ucs4_string u4_str = utils::string_to_ucs4string(str);
ucs4::string u4_str = utils::string_to_ucs4string(str);
if(u4_str.size() > size) {
u4_str.resize(size);
str = utils::ucs4string_to_string(u4_str);
@ -1029,7 +1029,7 @@ iterator& iterator::operator++()
return *this;
}
ucs4char iterator::operator*() const
ucs4::char_t iterator::operator*() const
{
return current_char;
}
@ -1089,10 +1089,10 @@ utf8::string lowercase(const utf8::string& s)
std::string res;
for(;itor != utf8::iterator::end(s); ++itor) {
ucs4char uchar = *itor;
ucs4::char_t uchar = *itor;
// If wchar_t is less than 32 bits wide, we cannot apply towlower() to all codepoints
if(uchar <= static_cast<ucs4char>(std::numeric_limits<wchar_t>::max()) &&
uchar >= static_cast<ucs4char>(std::numeric_limits<wchar_t>::min()))
if(uchar <= static_cast<ucs4::char_t>(std::numeric_limits<wchar_t>::max()) &&
uchar >= static_cast<ucs4::char_t>(std::numeric_limits<wchar_t>::min()))
uchar = towlower(static_cast<wchar_t>(uchar));
res += utils::ucs4char_to_string(uchar);
}

28
src/serialization/string_utils.hpp
View File

@ -26,15 +26,19 @@
#include "SDL_types.h"
typedef Uint32 ucs4char;
typedef std::vector<ucs4char> ucs4_string;
namespace ucs4 {
typedef Uint32 char_t;
typedef std::vector<char_t> string;
}
namespace utf8 { typedef std::string string; }
/**
* For win32 API.
* On windows, wchar_t is defined as Uint16
* Wide strings are expected to be UTF-16
*/
typedef std::vector<wchar_t> utf16_string;
namespace utf16 {
typedef std::vector<wchar_t> string;
}
class t_string;
@ -317,11 +321,11 @@ bool isvalid_wildcard(const std::string &login);
typedef std::map< std::string, t_string > string_map;
std::string ucs4string_to_string(const ucs4_string &);
ucs4_string string_to_ucs4string(const std::string &);
std::string ucs4char_to_string(const ucs4char);
std::string ucs4string_to_string(const ucs4::string &);
ucs4::string string_to_ucs4string(const std::string &);
std::string ucs4char_to_string(const ucs4::char_t);
utf16_string ucs4string_to_utf16string(const ucs4_string &);
utf16::string ucs4string_to_utf16string(const ucs4::string &);
/**
* Truncates a string.
@ -358,10 +362,10 @@ namespace utf8 {
{
public:
typedef std::input_iterator_tag iterator_category;
typedef ucs4char value_type;
typedef ucs4::char_t value_type;
typedef ptrdiff_t difference_type;
typedef ucs4char* pointer;
typedef ucs4char& reference;
typedef ucs4::char_t* pointer;
typedef ucs4::char_t& reference;
iterator(const std::string& str);
iterator(std::string::const_iterator const &begin, std::string::const_iterator const &end);
@ -372,13 +376,13 @@ namespace utf8 {
bool operator==(const utf8::iterator& a) const;
bool operator!=(const utf8::iterator& a) const { return ! (*this == a); }
iterator& operator++();
ucs4char operator*() const;
ucs4::char_t operator*() const;
bool next_is_end();
const std::pair<std::string::const_iterator, std::string::const_iterator>& substr() const;
private:
void update();
ucs4char current_char;
ucs4::char_t current_char;
std::string::const_iterator string_end;
std::pair<std::string::const_iterator, std::string::const_iterator> current_substr;
};

4
src/server/player_network.cpp
View File

@ -29,10 +29,10 @@ const size_t max_message_length = 256;
void truncate_message(const simple_wml::string_span& str, simple_wml::node& message)
{
// testing for msg.size() is not sufficient but we're not getting false negatives
// and it's cheaper than always converting to ucs4_string.
// and it's cheaper than always converting to ucs4::string.
if(str.size() > static_cast<int>(chat_message::max_message_length)) {
std::string tmp(str.begin(), str.end());
// The string can contain utf-8 characters so truncate as ucs4_string otherwise
// The string can contain utf-8 characters so truncate as ucs4::string otherwise
// a corrupted utf-8 string can be returned.
utils::truncate_as_ucs4string(tmp, max_message_length);
message.set_attr_dup("message", tmp.c_str());

12
src/tests/test_serialization.cpp
View File

@ -46,24 +46,24 @@ BOOST_AUTO_TEST_CASE( utils_unicode_test )
BOOST_CHECK( utf8::truncate(unicode,3) == "\xC3\xBCni"); // "üni"
utf8::string apple_u8("apple");
ucs4_string apple_u4 = utils::string_to_ucs4string(apple_u8);
utf16_string apple_u16 = utils::ucs4string_to_utf16string(apple_u4);
ucs4::string apple_u4 = utils::string_to_ucs4string(apple_u8);
utf16::string apple_u16 = utils::ucs4string_to_utf16string(apple_u4);
BOOST_CHECK( apple_u4.size() == 5 );
BOOST_CHECK_EQUAL( apple_u8, utils::ucs4string_to_string(apple_u4) );
BOOST_CHECK_EQUAL( apple_u8.size(), apple_u16.size() );
ucs4_string water_u4;
ucs4::string water_u4;
water_u4.push_back(0x6C34);
utf8::string water_u8 = utils::ucs4string_to_string(water_u4);
utf16_string water_u16 = utils::ucs4string_to_utf16string(water_u4);
utf16::string water_u16 = utils::ucs4string_to_utf16string(water_u4);
BOOST_CHECK_EQUAL(water_u4[0], water_u16[0]);
BOOST_CHECK_EQUAL(water_u8, "\u6C34");
utf8::string nonbmp_u8("\U00010000");
ucs4_string nonbmp_u4 = utils::string_to_ucs4string(nonbmp_u8);
utf16_string nonbmp_u16 = utils::ucs4string_to_utf16string(nonbmp_u4);
ucs4::string nonbmp_u4 = utils::string_to_ucs4string(nonbmp_u8);
utf16::string nonbmp_u16 = utils::ucs4string_to_utf16string(nonbmp_u4);
BOOST_CHECK_EQUAL(nonbmp_u8.size(), 4);
BOOST_CHECK_EQUAL(nonbmp_u4[0], 0x10000);

8
src/text.cpp
View File

@ -199,12 +199,12 @@ unsigned ttext::insert_text(const unsigned offset, const std::string& text)
return len;
}
bool ttext::insert_unicode(const unsigned offset, ucs4char unicode)
bool ttext::insert_unicode(const unsigned offset, ucs4::char_t unicode)
{
return (insert_unicode(offset, ucs4_string(1, unicode)) == 1);
return (insert_unicode(offset, ucs4::string(1, unicode)) == 1);
}
unsigned ttext::insert_unicode(const unsigned offset, const ucs4_string& unicode)
unsigned ttext::insert_unicode(const unsigned offset, const ucs4::string& unicode)
{
const utf8::string insert = utils::ucs4string_to_string(unicode);
return insert_text(offset, insert);
@ -293,7 +293,7 @@ bool ttext::set_text(const std::string& text, const bool markedup)
if(markedup != markedup_text_ || text != text_) {
assert(layout_);
const ucs4_string wide = utils::string_to_ucs4string(text);
const ucs4::string wide = utils::string_to_ucs4string(text);
const std::string narrow = utils::ucs4string_to_string(wide);
if(text != narrow) {
ERR_GUI_L << "ttext::" << __func__

4
src/text.hpp
View File

@ -102,7 +102,7 @@ public:
*
* @returns True upon success, false otherwise.
*/
bool insert_unicode(const unsigned offset, ucs4char unicode);
bool insert_unicode(const unsigned offset, ucs4::char_t unicode);
/**
* Inserts unicode text.
@ -113,7 +113,7 @@ public:
* @returns The number of characters inserted.
*/
unsigned insert_unicode(
const unsigned offset, const ucs4_string& unicode);
const unsigned offset, const ucs4::string& unicode);
/***** ***** ***** ***** Font flags ***** ***** ***** *****/

22
src/widgets/textbox.cpp
View File

@ -94,7 +94,7 @@ void textbox::append_text(const std::string& text, bool auto_scroll, const SDL_C
return;
}
const bool is_at_bottom = get_position() == get_max_position();
const ucs4_string& wtext = utils::string_to_ucs4string(text);
const ucs4::string& wtext = utils::string_to_ucs4string(text);
const surface new_text = add_text_line(wtext, color);
surface new_surface = create_compatible_surface(text_image_,std::max<size_t>(text_image_->w,new_text->w),text_image_->h+new_text->h);
@ -285,7 +285,7 @@ void textbox::scroll(unsigned int pos)
set_dirty(true);
}
surface textbox::add_text_line(const ucs4_string& text, const SDL_Color& color)
surface textbox::add_text_line(const ucs4::string& text, const SDL_Color& color)
{
line_height_ = font::get_max_height(font_size_);
@ -302,11 +302,11 @@ surface textbox::add_text_line(const ucs4_string& text, const SDL_Color& color)
// some more complex scripts (that is, RTL languages). This part of the work should
// actually be done by the font-rendering system.
std::string visible_string;
ucs4_string wrapped_text;
ucs4::string wrapped_text;
ucs4_string::const_iterator backup_itor = text.end();
ucs4::string::const_iterator backup_itor = text.end();
ucs4_string::const_iterator itor = text.begin();
ucs4::string::const_iterator itor = text.begin();
while(itor != text.end()) {
//If this is a space, save copies of the current state so we can roll back
if(char(*itor) == ' ') {
@ -332,7 +332,7 @@ surface textbox::add_text_line(const ucs4_string& text, const SDL_Color& color)
}
}
backup_itor = text.end();
wrapped_text.push_back(ucs4char('\n'));
wrapped_text.push_back(ucs4::char_t('\n'));
char_x_.push_back(0);
char_y_.push_back(char_y_.back() + line_height_);
visible_string = "";
@ -385,7 +385,7 @@ void textbox::erase_selection()
if(!is_selection())
return;
ucs4_string::iterator itor = text_.begin() + std::min(selstart_, selend_);
ucs4::string::iterator itor = text_.begin() + std::min(selstart_, selend_);
text_.erase(itor, itor + abs(selend_ - selstart_));
cursor_ = std::min(selstart_, selend_);
selstart_ = selend_ = -1;
@ -579,9 +579,9 @@ void textbox::handle_event(const SDL_Event& event, bool was_forwarded)
}
#if SDL_VERSION_ATLEAST(2, 0, 0)
ucs4char character = key.scancode;
ucs4::char_t character = key.scancode;
#else
ucs4char character = key.unicode;
ucs4::char_t character = key.unicode;
#endif
//movement characters may have a "Unicode" field on some platforms, so ignore it.
@ -614,7 +614,7 @@ void textbox::handle_event(const SDL_Event& event, bool was_forwarded)
//cut off anything after the first newline
str.erase(std::find_if(str.begin(),str.end(),utils::isnewline),str.end());
ucs4_string s = utils::string_to_ucs4string(str);
ucs4::string s = utils::string_to_ucs4string(str);
if(text_.size() < max_size_) {
if(s.size() + text_.size() > max_size_) {
@ -635,7 +635,7 @@ void textbox::handle_event(const SDL_Event& event, bool was_forwarded)
const size_t beg = std::min<size_t>(size_t(selstart_),size_t(selend_));
const size_t end = std::max<size_t>(size_t(selstart_),size_t(selend_));
ucs4_string ws(text_.begin() + beg, text_.begin() + end);
ucs4::string ws(text_.begin() + beg, text_.begin() + end);
std::string s = utils::ucs4string_to_string(ws);
copy_to_clipboard(s, false);
}

6
src/widgets/textbox.hpp
View File

@ -55,13 +55,13 @@ protected:
virtual void scroll(unsigned int pos);
private:
virtual void handle_text_changed(const ucs4_string&) {}
virtual void handle_text_changed(const ucs4::string&) {}
size_t max_size_;
int font_size_;
ucs4_string text_;
ucs4::string text_;
// mutable unsigned int firstOnScreen_;
int cursor_;
@ -100,7 +100,7 @@ private:
void draw_cursor(int pos, CVideo &video) const;
void update_text_cache(bool reset = false, const SDL_Color& color =font::NORMAL_COLOR);
surface add_text_line(const ucs4_string& text, const SDL_Color& color =font::NORMAL_COLOR);
surface add_text_line(const ucs4::string& text, const SDL_Color& color =font::NORMAL_COLOR);
bool is_selection();
void erase_selection();

4
src/windows_tray_notification.cpp
View File

@ -178,8 +178,8 @@ void windows_tray_notification::switch_to_wesnoth_window()
std::wstring windows_tray_notification::string_to_wstring(const std::string& string, size_t maxlength)
{
const utils::ucs4_string u4_string = utils::string_to_ucs4string(string);
utils::utf16_string u16_string = utils::ucs4string_to_utf16string(u4_string);
const ucs4::string u4_string = utils::string_to_ucs4string(string);
utf16::string u16_string = utils::ucs4string_to_utf16string(u4_string);
if(u16_string.size() > maxlength) {
if((u16_string[maxlength-1] & 0xDC00) == 0xD800)
u16_string.resize(maxlength - 1);