OpenTTD Source: src/string.cpp Source File

OpenTTD Source 13.1

string.cpp
Go to the documentation of this file.
1 /*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6 */
7
10 #include "stdafx.h"
11 #include "debug.h"
12 #include "core/alloc_func.hpp"
13 #include "core/math_func.hpp"
14 #include "string_func.h"
15 #include "string_base.h"
16
17 #include "table/control_codes.h"
18
19 #include <stdarg.h>
20 #include <ctype.h> /* required for tolower() */
21 #include <sstream>
22 #include <iomanip>
23
24 #ifdef _MSC_VER
25 #include <errno.h> // required by vsnprintf implementation for MSVC
26 #endif
27
28 #ifdef _WIN32
29 #include "os/windows/win32.h"
30 #endif
31
32 #ifdef WITH_UNISCRIBE
33 #include "os/windows/string_uniscribe.h"
34 #endif
35
36 #ifdef WITH_ICU_I18N
37 /* Required by strnatcmp. */
38 #include <unicode/ustring.h>
39 #include "language.h"
40 #include "gfx_func.h"
41 #endif /* WITH_ICU_I18N */
42
43 #if defined(WITH_COCOA)
44 #include "os/macosx/string_osx.h"
45 #endif
46
47 /* The function vsnprintf is used internally to perform the required formatting
48 * tasks. As such this one must be allowed, and makes sure it's terminated. */
49 #include "safeguards.h"
50 #undef vsnprintf
51
62 int CDECL vseprintf(char *str, const char *last, const char *format, va_list ap)
63 {
64 ptrdiff_t diff = last - str;
65 if (diff < 0) return 0;
66 return std::min(static_cast<int>(diff), vsnprintf(str, diff + 1, format, ap));
67 }
68
85 char *strecat(char *dst, const char *src, const char *last)
86 {
87 assert(dst <= last);
88 while (*dst != '\0') {
89 if (dst == last) return dst;
90 dst++;
91 }
92
93 return strecpy(dst, src, last);
94 }
95
96
113 char *strecpy(char *dst, const char *src, const char *last)
114 {
115 assert(dst <= last);
116 while (dst != last && *src != '\0') {
117 *dst++ = *src++;
118 }
119 *dst = '\0';
120
121 if (dst == last && *src != '\0') {
122 #if defined(STRGEN) || defined(SETTINGSGEN)
123 error("String too long for destination buffer");
124 #else /* STRGEN || SETTINGSGEN */
125 Debug(misc, 0, "String too long for destination buffer");
126 #endif /* STRGEN || SETTINGSGEN */
127 }
128 return dst;
129 }
130
138 char *stredup(const char *s, const char *last)
139 {
140 size_t len = last == nullptr ? strlen(s) : ttd_strnlen(s, last - s + 1);
141 char *tmp = CallocT<char>(len + 1);
142 memcpy(tmp, s, len);
143 return tmp;
144 }
145
151 char *CDECL str_fmt(const char *str, ...)
152 {
153 char buf[4096];
154 va_list va;
155
156 va_start(va, str);
157 int len = vseprintf(buf, lastof(buf), str, va);
158 va_end(va);
159 char *p = MallocT<char>(len + 1);
160 memcpy(p, buf, len + 1);
161 return p;
162 }
163
169 std::string FormatArrayAsHex(span<const byte> data)
170 {
171 std::ostringstream ss;
172 ss << std::uppercase << std::setfill('0') << std::setw(2) << std::hex;
173
174 for (auto b : data) {
175 ss << b;
176 }
177
178 return ss.str();
179 }
180
187 void str_fix_scc_encoded(char *str, const char *last)
188 {
189 while (str <= last && *str != '\0') {
190 size_t len = Utf8EncodedCharLen(*str);
191 if ((len == 0 && str + 4 > last) || str + len > last) break;
192
193 WChar c;
194 Utf8Decode(&c, str);
195 if (c == '\0') break;
196
197 if (c == 0xE028 || c == 0xE02A) {
198 c = SCC_ENCODED;
199 }
200 str += Utf8Encode(str, c);
201 }
202 *str = '\0';
203 }
204
205
206 template <class T>
207 static void StrMakeValidInPlace(T &dst, const char *str, const char *last, StringValidationSettings settings)
208 {
209 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
210
211 while (str <= last && *str != '\0') {
212 size_t len = Utf8EncodedCharLen(*str);
213 WChar c;
214 /* If the first byte does not look like the first byte of an encoded
215 * character, i.e. encoded length is 0, then this byte is definitely bad
216 * and it should be skipped.
217 * When the first byte looks like the first byte of an encoded character,
218 * then the remaining bytes in the string are checked whether the whole
219 * encoded character can be there. If that is not the case, this byte is
220 * skipped.
221 * Finally we attempt to decode the encoded character, which does certain
222 * extra validations to see whether the correct number of bytes were used
223 * to encode the character. If that is not the case, the byte is probably
224 * invalid and it is skipped. We could emit a question mark, but then the
225 * logic below cannot just copy bytes, it would need to re-encode the
226 * decoded characters as the length in bytes may have changed.
227 *
228 * The goals here is to get as much valid Utf8 encoded characters from the
229 * source string to the destination string.
230 *
231 * Note: a multi-byte encoded termination ('\0') will trigger the encoded
232 * char length and the decoded length to differ, so it will be ignored as
233 * invalid character data. If it were to reach the termination, then we
234 * would also reach the "last" byte of the string and a normal '\0'
235 * termination will be placed after it.
236 */
237 if (len == 0 || str + len > last || len != Utf8Decode(&c, str)) {
238 /* Maybe the next byte is still a valid character? */
239 str++;
240 continue;
241 }
242
243 if ((IsPrintable(c) && (c < SCC_SPRITE_START || c > SCC_SPRITE_END)) || ((settings & SVS_ALLOW_CONTROL_CODE) != 0 && c == SCC_ENCODED)) {
244 /* Copy the character back. Even if dst is current the same as str
245 * (i.e. no characters have been changed) this is quicker than
246 * moving the pointers ahead by len */
247 do {
248 *dst++ = *str++;
249 } while (--len != 0);
250 } else if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\n') {
251 *dst++ = *str++;
252 } else {
253 if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\r' && str[1] == '\n') {
254 str += len;
255 continue;
256 }
257 /* Replace the undesirable character with a question mark */
258 str += len;
259 if ((settings & SVS_REPLACE_WITH_QUESTION_MARK) != 0) *dst++ = '?';
260 }
261 }
262
263 /* String termination, if needed, is left to the caller of this function. */
264 }
265
273 void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
274 {
275 char *dst = str;
276 StrMakeValidInPlace(dst, str, last, settings);
277 *dst = '\0';
278 }
279
287 void StrMakeValidInPlace(char *str, StringValidationSettings settings)
288 {
289 /* We know it is '\0' terminated. */
290 StrMakeValidInPlace(str, str + strlen(str), settings);
291 }
292
299 std::string StrMakeValid(const std::string &str, StringValidationSettings settings)
300 {
301 auto buf = str.data();
302 auto last = buf + str.size();
303
304 std::ostringstream dst;
305 std::ostreambuf_iterator<char> dst_iter(dst);
306 StrMakeValidInPlace(dst_iter, buf, last, settings);
307
308 return dst.str();
309 }
310
318 bool StrValid(const char *str, const char *last)
319 {
320 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
321
322 while (str <= last && *str != '\0') {
323 size_t len = Utf8EncodedCharLen(*str);
324 /* Encoded length is 0 if the character isn't known.
325 * The length check is needed to prevent Utf8Decode to read
326 * over the terminating '\0' if that happens to be placed
327 * within the encoding of an UTF8 character. */
328 if (len == 0 || str + len > last) return false;
329
330 WChar c;
331 len = Utf8Decode(&c, str);
332 if (!IsPrintable(c) || (c >= SCC_SPRITE_START && c <= SCC_SPRITE_END)) {
333 return false;
334 }
335
336 str += len;
337 }
338
339 return *str == '\0';
340 }
341
348 static void StrLeftTrimInPlace(std::string &str)
349 {
350 size_t pos = str.find_first_not_of(' ');
351 str.erase(0, pos);
352 }
353
360 static void StrRightTrimInPlace(std::string &str)
361 {
362 size_t pos = str.find_last_not_of(' ');
363 if (pos != std::string::npos) str.erase(pos + 1);
364 }
365
373 void StrTrimInPlace(std::string &str)
374 {
375 StrLeftTrimInPlace(str);
376 StrRightTrimInPlace(str);
377 }
378
385 bool StrStartsWith(const std::string_view str, const std::string_view prefix)
386 {
387 size_t prefix_len = prefix.size();
388 if (str.size() < prefix_len) return false;
389 return str.compare(0, prefix_len, prefix, 0, prefix_len) == 0;
390 }
391
398 bool StrEndsWith(const std::string_view str, const std::string_view suffix)
399 {
400 size_t suffix_len = suffix.size();
401 if (str.size() < suffix_len) return false;
402 return str.compare(str.size() - suffix_len, suffix_len, suffix, 0, suffix_len) == 0;
403 }
404
405
407 void str_strip_colours(char *str)
408 {
409 char *dst = str;
410 WChar c;
411 size_t len;
412
413 for (len = Utf8Decode(&c, str); c != '\0'; len = Utf8Decode(&c, str)) {
414 if (c < SCC_BLUE || c > SCC_BLACK) {
415 /* Copy the character back. Even if dst is current the same as str
416 * (i.e. no characters have been changed) this is quicker than
417 * moving the pointers ahead by len */
418 do {
419 *dst++ = *str++;
420 } while (--len != 0);
421 } else {
422 /* Just skip (strip) the colour codes */
423 str += len;
424 }
425 }
426 *dst = '\0';
427 }
428
435 size_t Utf8StringLength(const char *s)
436 {
437 size_t len = 0;
438 const char *t = s;
439 while (Utf8Consume(&t) != 0) len++;
440 return len;
441 }
442
449 size_t Utf8StringLength(const std::string &str)
450 {
451 return Utf8StringLength(str.c_str());
452 }
453
465 bool strtolower(char *str)
466 {
467 bool changed = false;
468 for (; *str != '\0'; str++) {
469 char new_str = tolower(*str);
470 changed |= new_str != *str;
471 *str = new_str;
472 }
473 return changed;
474 }
475
476 bool strtolower(std::string &str, std::string::size_type offs)
477 {
478 bool changed = false;
479 for (auto ch = str.begin() + offs; ch != str.end(); ++ch) {
480 auto new_ch = static_cast<char>(tolower(static_cast<unsigned char>(*ch)));
481 changed |= new_ch != *ch;
482 *ch = new_ch;
483 }
484 return changed;
485 }
486
494 bool IsValidChar(WChar key, CharSetFilter afilter)
495 {
496 switch (afilter) {
497 case CS_ALPHANUMERAL: return IsPrintable(key);
498 case CS_NUMERAL: return (key >= '0' && key <= '9');
499 case CS_NUMERAL_SPACE: return (key >= '0' && key <= '9') || key == ' ';
500 case CS_NUMERAL_SIGNED: return (key >= '0' && key <= '9') || key == '-';
501 case CS_ALPHA: return IsPrintable(key) && !(key >= '0' && key <= '9');
502 case CS_HEXADECIMAL: return (key >= '0' && key <= '9') || (key >= 'a' && key <= 'f') || (key >= 'A' && key <= 'F');
503 default: NOT_REACHED();
504 }
505 }
506
507 #ifdef _WIN32
508 #if defined(_MSC_VER) && _MSC_VER < 1900
509
516 int CDECL vsnprintf(char *str, size_t size, const char *format, va_list ap)
517 {
518 if (size == 0) return 0;
519
520 errno = 0;
521 int ret = _vsnprintf(str, size, format, ap);
522
523 if (ret < 0) {
524 if (errno != ERANGE) {
525 /* There's a formatting error, better get that looked
526 * at properly instead of ignoring it. */
527 NOT_REACHED();
528 }
529 } else if ((size_t)ret < size) {
530 /* The buffer is big enough for the number of
531 * characters stored (excluding null), i.e.
532 * the string has been null-terminated. */
533 return ret;
534 }
535
536 /* The buffer is too small for _vsnprintf to write the
537 * null-terminator at its end and return size. */
538 str[size - 1] = '\0';
539 return (int)size;
540 }
541 #endif /* _MSC_VER */
542
543 #endif /* _WIN32 */
544
554 int CDECL seprintf(char *str, const char *last, const char *format, ...)
555 {
556 va_list ap;
557
558 va_start(ap, format);
559 int ret = vseprintf(str, last, format, ap);
560 va_end(ap);
561 return ret;
562 }
563
564
572 char *md5sumToString(char *buf, const char *last, const uint8 md5sum[16])
573 {
574 char *p = buf;
575
576 for (uint i = 0; i < 16; i++) {
577 p += seprintf(p, last, "%02X", md5sum[i]);
578 }
579
580 return p;
581 }
582
583
584 /* UTF-8 handling routines */
585
586
593 size_t Utf8Decode(WChar *c, const char *s)
594 {
595 assert(c != nullptr);
596
597 if (!HasBit(s[0], 7)) {
598 /* Single byte character: 0xxxxxxx */
599 *c = s[0];
600 return 1;
601 } else if (GB(s[0], 5, 3) == 6) {
602 if (IsUtf8Part(s[1])) {
603 /* Double byte character: 110xxxxx 10xxxxxx */
604 *c = GB(s[0], 0, 5) << 6 | GB(s[1], 0, 6);
605 if (*c >= 0x80) return 2;
606 }
607 } else if (GB(s[0], 4, 4) == 14) {
608 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
609 /* Triple byte character: 1110xxxx 10xxxxxx 10xxxxxx */
610 *c = GB(s[0], 0, 4) << 12 | GB(s[1], 0, 6) << 6 | GB(s[2], 0, 6);
611 if (*c >= 0x800) return 3;
612 }
613 } else if (GB(s[0], 3, 5) == 30) {
614 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
615 /* 4 byte character: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
616 *c = GB(s[0], 0, 3) << 18 | GB(s[1], 0, 6) << 12 | GB(s[2], 0, 6) << 6 | GB(s[3], 0, 6);
617 if (*c >= 0x10000 && *c <= 0x10FFFF) return 4;
618 }
619 }
620
621 /* Debug(misc, 1, "[utf8] invalid UTF-8 sequence"); */
622 *c = '?';
623 return 1;
624 }
625
626
634 template <class T>
635 inline size_t Utf8Encode(T buf, WChar c)
636 {
637 if (c < 0x80) {
638 *buf = c;
639 return 1;
640 } else if (c < 0x800) {
641 *buf++ = 0xC0 + GB(c, 6, 5);
642 *buf = 0x80 + GB(c, 0, 6);
643 return 2;
644 } else if (c < 0x10000) {
645 *buf++ = 0xE0 + GB(c, 12, 4);
646 *buf++ = 0x80 + GB(c, 6, 6);
647 *buf = 0x80 + GB(c, 0, 6);
648 return 3;
649 } else if (c < 0x110000) {
650 *buf++ = 0xF0 + GB(c, 18, 3);
651 *buf++ = 0x80 + GB(c, 12, 6);
652 *buf++ = 0x80 + GB(c, 6, 6);
653 *buf = 0x80 + GB(c, 0, 6);
654 return 4;
655 }
656
657 /* Debug(misc, 1, "[utf8] can't UTF-8 encode value 0x{:X}", c); */
658 *buf = '?';
659 return 1;
660 }
661
662 size_t Utf8Encode(char *buf, WChar c)
663 {
664 return Utf8Encode<char *>(buf, c);
665 }
666
667 size_t Utf8Encode(std::ostreambuf_iterator<char> &buf, WChar c)
668 {
669 return Utf8Encode<std::ostreambuf_iterator<char> &>(buf, c);
670 }
671
679 size_t Utf8TrimString(char *s, size_t maxlen)
680 {
681 size_t length = 0;
682
683 for (const char *ptr = strchr(s, '\0'); *s != '\0';) {
684 size_t len = Utf8EncodedCharLen(*s);
685 /* Silently ignore invalid UTF8 sequences, our only concern trimming */
686 if (len == 0) len = 1;
687
688 /* Take care when a hard cutoff was made for the string and
689 * the last UTF8 sequence is invalid */
690 if (length + len >= maxlen || (s + len > ptr)) break;
691 s += len;
692 length += len;
693 }
694
695 *s = '\0';
696 return length;
697 }
698
699 #ifdef DEFINE_STRCASESTR
700 char *strcasestr(const char *haystack, const char *needle)
701 {
702 size_t hay_len = strlen(haystack);
703 size_t needle_len = strlen(needle);
704 while (hay_len >= needle_len) {
705 if (strncasecmp(haystack, needle, needle_len) == 0) return const_cast<char *>(haystack);
706
707 haystack++;
708 hay_len--;
709 }
710
711 return nullptr;
712 }
713 #endif /* DEFINE_STRCASESTR */
714
723 static const char *SkipGarbage(const char *str)
724 {
725 while (*str != '\0' && (*str < '0' || IsInsideMM(*str, ';', '@' + 1) || IsInsideMM(*str, '[', '`' + 1) || IsInsideMM(*str, '{', '~' + 1))) str++;
726 return str;
727 }
728
737 int strnatcmp(const char *s1, const char *s2, bool ignore_garbage_at_front)
738 {
739 if (ignore_garbage_at_front) {
740 s1 = SkipGarbage(s1);
741 s2 = SkipGarbage(s2);
742 }
743
744 #ifdef WITH_ICU_I18N
745 if (_current_collator) {
746 UErrorCode status = U_ZERO_ERROR;
747 int result = _current_collator->compareUTF8(s1, s2, status);
748 if (U_SUCCESS(status)) return result;
749 }
750 #endif /* WITH_ICU_I18N */
751
752 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
753 int res = OTTDStringCompare(s1, s2);
754 if (res != 0) return res - 2; // Convert to normal C return values.
755 #endif
756
757 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
758 int res = MacOSStringCompare(s1, s2);
759 if (res != 0) return res - 2; // Convert to normal C return values.
760 #endif
761
762 /* Do a normal comparison if ICU is missing or if we cannot create a collator. */
763 return strcasecmp(s1, s2);
764 }
765
766 #ifdef WITH_UNISCRIBE
767
768 /* static */ StringIterator *StringIterator::Create()
769 {
770 return new UniscribeStringIterator();
771 }
772
773 #elif defined(WITH_ICU_I18N)
774
775 #include <unicode/utext.h>
776 #include <unicode/brkiter.h>
777
779 class IcuStringIterator : public StringIterator
780 {
781 icu::BreakIterator *char_itr;
782 icu::BreakIterator *word_itr;
783
784 std::vector<UChar> utf16_str;
785 std::vector<size_t> utf16_to_utf8;
786
787 public:
788 IcuStringIterator() : char_itr(nullptr), word_itr(nullptr)
789 {
790 UErrorCode status = U_ZERO_ERROR;
791 this->char_itr = icu::BreakIterator::createCharacterInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
792 this->word_itr = icu::BreakIterator::createWordInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
793
794 this->utf16_str.push_back('\0');
795 this->utf16_to_utf8.push_back(0);
796 }
797
798 ~IcuStringIterator() override
799 {
800 delete this->char_itr;
801 delete this->word_itr;
802 }
803
804 void SetString(const char *s) override
805 {
806 const char *string_base = s;
807
808 /* Unfortunately current ICU versions only provide rudimentary support
809 * for word break iterators (especially for CJK languages) in combination
810 * with UTF-8 input. As a work around we have to convert the input to
811 * UTF-16 and create a mapping back to UTF-8 character indices. */
812 this->utf16_str.clear();
813 this->utf16_to_utf8.clear();
814
815 while (*s != '\0') {
816 size_t idx = s - string_base;
817
818 WChar c = Utf8Consume(&s);
819 if (c < 0x10000) {
820 this->utf16_str.push_back((UChar)c);
821 } else {
822 /* Make a surrogate pair. */
823 this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
824 this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
825 this->utf16_to_utf8.push_back(idx);
826 }
827 this->utf16_to_utf8.push_back(idx);
828 }
829 this->utf16_str.push_back('\0');
830 this->utf16_to_utf8.push_back(s - string_base);
831
832 UText text = UTEXT_INITIALIZER;
833 UErrorCode status = U_ZERO_ERROR;
834 utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
835 this->char_itr->setText(&text, status);
836 this->word_itr->setText(&text, status);
837 this->char_itr->first();
838 this->word_itr->first();
839 }
840
841 size_t SetCurPosition(size_t pos) override
842 {
843 /* Convert incoming position to an UTF-16 string index. */
844 uint utf16_pos = 0;
845 for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
846 if (this->utf16_to_utf8[i] == pos) {
847 utf16_pos = i;
848 break;
849 }
850 }
851
852 /* isBoundary has the documented side-effect of setting the current
853 * position to the first valid boundary equal to or greater than
854 * the passed value. */
855 this->char_itr->isBoundary(utf16_pos);
856 return this->utf16_to_utf8[this->char_itr->current()];
857 }
858
859 size_t Next(IterType what) override
860 {
861 int32_t pos;
862 switch (what) {
863 case ITER_CHARACTER:
864 pos = this->char_itr->next();
865 break;
866
867 case ITER_WORD:
868 pos = this->word_itr->following(this->char_itr->current());
869 /* The ICU word iterator considers both the start and the end of a word a valid
870 * break point, but we only want word starts. Move to the next location in
871 * case the new position points to whitespace. */
872 while (pos != icu::BreakIterator::DONE &&
873 IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
874 int32_t new_pos = this->word_itr->next();
875 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
876 * even though the iterator wasn't at the end of the string before. */
877 if (new_pos == icu::BreakIterator::DONE) break;
878 pos = new_pos;
879 }
880
881 this->char_itr->isBoundary(pos);
882 break;
883
884 default:
885 NOT_REACHED();
886 }
887
888 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
889 }
890
891 size_t Prev(IterType what) override
892 {
893 int32_t pos;
894 switch (what) {
895 case ITER_CHARACTER:
896 pos = this->char_itr->previous();
897 break;
898
899 case ITER_WORD:
900 pos = this->word_itr->preceding(this->char_itr->current());
901 /* The ICU word iterator considers both the start and the end of a word a valid
902 * break point, but we only want word starts. Move to the previous location in
903 * case the new position points to whitespace. */
904 while (pos != icu::BreakIterator::DONE &&
905 IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
906 int32_t new_pos = this->word_itr->previous();
907 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
908 * even though the iterator wasn't at the start of the string before. */
909 if (new_pos == icu::BreakIterator::DONE) break;
910 pos = new_pos;
911 }
912
913 this->char_itr->isBoundary(pos);
914 break;
915
916 default:
917 NOT_REACHED();
918 }
919
920 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
921 }
922 };
923
924 /* static */ StringIterator *StringIterator::Create()
925 {
926 return new IcuStringIterator();
927 }
928
929 #else
930
932 class DefaultStringIterator : public StringIterator
933 {
934 const char *string;
935 size_t len;
936 size_t cur_pos;
937
938 public:
939 DefaultStringIterator() : string(nullptr), len(0), cur_pos(0)
940 {
941 }
942
943 virtual void SetString(const char *s)
944 {
945 this->string = s;
946 this->len = strlen(s);
947 this->cur_pos = 0;
948 }
949
950 virtual size_t SetCurPosition(size_t pos)
951 {
952 assert(this->string != nullptr && pos <= this->len);
953 /* Sanitize in case we get a position inside an UTF-8 sequence. */
954 while (pos > 0 && IsUtf8Part(this->string[pos])) pos--;
955 return this->cur_pos = pos;
956 }
957
958 virtual size_t Next(IterType what)
959 {
960 assert(this->string != nullptr);
961
962 /* Already at the end? */
963 if (this->cur_pos >= this->len) return END;
964
965 switch (what) {
966 case ITER_CHARACTER: {
967 WChar c;
968 this->cur_pos += Utf8Decode(&c, this->string + this->cur_pos);
969 return this->cur_pos;
970 }
971
972 case ITER_WORD: {
973 WChar c;
974 /* Consume current word. */
975 size_t offs = Utf8Decode(&c, this->string + this->cur_pos);
976 while (this->cur_pos < this->len && !IsWhitespace(c)) {
977 this->cur_pos += offs;
978 offs = Utf8Decode(&c, this->string + this->cur_pos);
979 }
980 /* Consume whitespace to the next word. */
981 while (this->cur_pos < this->len && IsWhitespace(c)) {
982 this->cur_pos += offs;
983 offs = Utf8Decode(&c, this->string + this->cur_pos);
984 }
985
986 return this->cur_pos;
987 }
988
989 default:
990 NOT_REACHED();
991 }
992
993 return END;
994 }
995
996 virtual size_t Prev(IterType what)
997 {
998 assert(this->string != nullptr);
999
1000 /* Already at the beginning? */
1001 if (this->cur_pos == 0) return END;
1002
1003 switch (what) {
1004 case ITER_CHARACTER:
1005 return this->cur_pos = Utf8PrevChar(this->string + this->cur_pos) - this->string;
1006
1007 case ITER_WORD: {
1008 const char *s = this->string + this->cur_pos;
1009 WChar c;
1010 /* Consume preceding whitespace. */
1011 do {
1012 s = Utf8PrevChar(s);
1013 Utf8Decode(&c, s);
1014 } while (s > this->string && IsWhitespace(c));
1015 /* Consume preceding word. */
1016 while (s > this->string && !IsWhitespace(c)) {
1017 s = Utf8PrevChar(s);
1018 Utf8Decode(&c, s);
1019 }
1020 /* Move caret back to the beginning of the word. */
1021 if (IsWhitespace(c)) Utf8Consume(&s);
1022
1023 return this->cur_pos = s - this->string;
1024 }
1025
1026 default:
1027 NOT_REACHED();
1028 }
1029
1030 return END;
1031 }
1032 };
1033
1034 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
1035 /* static */ StringIterator *StringIterator::Create()
1036 {
1037 StringIterator *i = OSXStringIterator::Create();
1038 if (i != nullptr) return i;
1039
1040 return new DefaultStringIterator();
1041 }
1042 #else
1043 /* static */ StringIterator *StringIterator::Create()
1044 {
1045 return new DefaultStringIterator();
1046 }
1047 #endif /* defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN) */
1048
1049 #endif
IsInsideMM
static constexpr bool IsInsideMM(const T x, const size_t min, const size_t max) noexcept
Checks if a value is in an interval.
Definition: math_func.hpp:204
StringIterator::Prev
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
IcuStringIterator::utf16_to_utf8
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
Definition: string.cpp:785
WChar
char32_t WChar
Type for wide characters, i.e.
Definition: string_type.h:36
SVS_ALLOW_NEWLINE
@ SVS_ALLOW_NEWLINE
Allow newlines.
Definition: string_type.h:52
GB
static uint GB(const T x, const uint8 s, const uint8 n)
Fetch n bits from x, started at bit s.
Definition: bitmath_func.hpp:32
StringIterator::IterType
IterType
Type of the iterator.
Definition: string_base.h:17
strtolower
bool strtolower(char *str)
Convert a given ASCII string to lowercase.
Definition: string.cpp:465
win32.h
StrRightTrimInPlace
static void StrRightTrimInPlace(std::string &str)
Trim the spaces from the end of given string in place, i.e.
Definition: string.cpp:360
StringIterator::END
static const size_t END
Sentinel to indicate end-of-iteration.
Definition: string_base.h:23
math_func.hpp
str_fix_scc_encoded
void str_fix_scc_encoded(char *str, const char *last)
Scan the string for old values of SCC_ENCODED and fix it to it's new, static value.
Definition: string.cpp:187
HasBit
static bool HasBit(const T x, const uint8 y)
Checks if a bit in a value is set.
Definition: bitmath_func.hpp:103
IcuStringIterator::word_itr
icu::BreakIterator * word_itr
ICU iterator for words.
Definition: string.cpp:782
_current_collator
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
Definition: strings.cpp:52
Utf8Encode
size_t Utf8Encode(T buf, WChar c)
Encode a unicode character and place it in the buffer.
Definition: string.cpp:635
StringIterator::Next
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
CS_ALPHA
@ CS_ALPHA
Only alphabetic values.
Definition: string_type.h:31
UniscribeStringIterator
String iterator using Uniscribe as a backend.
Definition: string_uniscribe.h:67
Utf16DecodeChar
static WChar Utf16DecodeChar(const uint16 *c)
Decode an UTF-16 character.
Definition: string_func.h:213
StringIterator::SetString
virtual void SetString(const char *s)=0
Set a new iteration string.
IcuStringIterator::char_itr
icu::BreakIterator * char_itr
ICU iterator for characters.
Definition: string.cpp:781
StringIterator::ITER_CHARACTER
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
Definition: string_base.h:18
control_codes.h
IcuStringIterator::Next
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
Definition: string.cpp:859
string_osx.h
gfx_func.h
FormatArrayAsHex
std::string FormatArrayAsHex(span< const byte > data)
Format a byte array into a continuous hex string.
Definition: string.cpp:169
Utf8StringLength
size_t Utf8StringLength(const char *s)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
Definition: string.cpp:435
StrValid
bool StrValid(const char *str, const char *last)
Checks whether the given string is valid, i.e.
Definition: string.cpp:318
StringIterator::Create
static StringIterator * Create()
Create a new iterator instance.
Definition: string.cpp:924
SVS_ALLOW_CONTROL_CODE
@ SVS_ALLOW_CONTROL_CODE
Allow the special control codes.
Definition: string_type.h:53
StringIterator
Class for iterating over different kind of parts of a string.
Definition: string_base.h:14
IcuStringIterator::SetCurPosition
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
Definition: string.cpp:841
span< const byte >
StringIterator::SetCurPosition
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
StrMakeValidInPlace
void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
Definition: string.cpp:273
StrTrimInPlace
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
Definition: string.cpp:373
str_strip_colours
void str_strip_colours(char *str)
Scans the string for colour codes and strips them.
Definition: string.cpp:407
StrStartsWith
bool StrStartsWith(const std::string_view str, const std::string_view prefix)
Check whether the given string starts with the given prefix.
Definition: string.cpp:385
_current_language
const LanguageMetadata * _current_language
The currently loaded language.
Definition: strings.cpp:47
safeguards.h
IcuStringIterator::utf16_str
std::vector< UChar > utf16_str
UTF-16 copy of the string.
Definition: string.cpp:784
IsValidChar
bool IsValidChar(WChar key, CharSetFilter afilter)
Only allow certain keys.
Definition: string.cpp:494
ttd_strnlen
static size_t ttd_strnlen(const char *str, size_t maxlen)
Get the length of a string, within a limited buffer.
Definition: string_func.h:79
settings
fluid_settings_t * settings
FluidSynth settings handle.
Definition: fluidsynth.cpp:21
vseprintf
int CDECL vseprintf(char *str, const char *last, const char *format, va_list ap)
Safer implementation of vsnprintf; same as vsnprintf except:
Definition: string.cpp:62
StringIterator::ITER_WORD
@ ITER_WORD
Iterate over words.
Definition: string_base.h:19
language.h
stdafx.h
CS_ALPHANUMERAL
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
Definition: string_type.h:27
LanguagePackHeader::isocode
char isocode[16]
the ISO code for the language (not country code)
Definition: language.h:31
StringValidationSettings
StringValidationSettings
Settings for the string validation.
Definition: string_type.h:49
StrEndsWith
bool StrEndsWith(const std::string_view str, const std::string_view suffix)
Check whether the given string ends with the given suffix.
Definition: string.cpp:398
Utf8Decode
size_t Utf8Decode(WChar *c, const char *s)
Decode and consume the next UTF-8 encoded character.
Definition: string.cpp:593
string_func.h
StrMakeValid
std::string StrMakeValid(const std::string &str, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
Definition: string.cpp:299
str_fmt
char *CDECL str_fmt(const char *str,...)
Format, "printf", into a newly allocated string.
Definition: string.cpp:151
alloc_func.hpp
IcuStringIterator::SetString
void SetString(const char *s) override
Set a new iteration string.
Definition: string.cpp:804
StrLeftTrimInPlace
static void StrLeftTrimInPlace(std::string &str)
Trim the spaces from the begin of given string in place, i.e.
Definition: string.cpp:348
IcuStringIterator
String iterator using ICU as a backend.
Definition: string.cpp:779
seprintf
int CDECL seprintf(char *str, const char *last, const char *format,...)
Safer implementation of snprintf; same as snprintf except:
Definition: string.cpp:554
CS_NUMERAL_SIGNED
@ CS_NUMERAL_SIGNED
Only numbers and '-' for negative values.
Definition: string_type.h:30
stredup
char * stredup(const char *s, const char *last)
Create a duplicate of the given string.
Definition: string.cpp:138
error
void CDECL error(const char *s,...)
Error handling for fatal non-user errors.
Definition: openttd.cpp:134
Debug
#define Debug(name, level, format_string,...)
Ouptut a line of debugging information.
Definition: debug.h:37
Utf8TrimString
size_t Utf8TrimString(char *s, size_t maxlen)
Properly terminate an UTF8 string to some maximum length.
Definition: string.cpp:679
CS_NUMERAL_SPACE
@ CS_NUMERAL_SPACE
Only numbers and spaces.
Definition: string_type.h:29
SkipGarbage
static const char * SkipGarbage(const char *str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
Definition: string.cpp:723
IsWhitespace
static bool IsWhitespace(WChar c)
Check whether UNICODE character is whitespace or not, i.e.
Definition: string_func.h:260
strnatcmp
int strnatcmp(const char *s1, const char *s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
Definition: string.cpp:737
md5sumToString
char * md5sumToString(char *buf, const char *last, const uint8 md5sum[16])
Convert the md5sum to a hexadecimal string representation.
Definition: string.cpp:572
strecpy
char * strecpy(char *dst, const char *src, const char *last)
Copies characters from one buffer to another.
Definition: string.cpp:113
CS_HEXADECIMAL
@ CS_HEXADECIMAL
Only hexadecimal characters.
Definition: string_type.h:32
strecat
char * strecat(char *dst, const char *src, const char *last)
Appends characters from one string to another.
Definition: string.cpp:85
MacOSStringCompare
int MacOSStringCompare(const char *s1, const char *s2)
Compares two strings using case insensitive natural sort.
Definition: string_osx.cpp:325
lastof
#define lastof(x)
Get the last element of an fixed size array.
Definition: stdafx.h:402
SVS_REPLACE_WITH_QUESTION_MARK
@ SVS_REPLACE_WITH_QUESTION_MARK
Replace the unknown/bad bits with question marks.
Definition: string_type.h:51
CS_NUMERAL
@ CS_NUMERAL
Only numeric ones.
Definition: string_type.h:28
CharSetFilter
CharSetFilter
Valid filter types for IsValidChar.
Definition: string_type.h:26
Utf8PrevChar
static char * Utf8PrevChar(char *s)
Retrieve the previous UNICODE character in an UTF-8 encoded string.
Definition: string_func.h:160
debug.h
string_uniscribe.h
Utf8EncodedCharLen
static int8 Utf8EncodedCharLen(char c)
Return the length of an UTF-8 encoded value based on a single char.
Definition: string_func.h:135
IcuStringIterator::Prev
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
Definition: string.cpp:891