Windows: encode and decode text as UTF-8 instead of trusting the locale
setlocale(LC_ALL, "") maps to the ANSI code page on Windows -- 936 (GBK)
on a Chinese system -- so wcrtomb() produces GBK where the protocol wants
UTF-8, and mbrtowc() reads UTF-8 as GBK. Typed Chinese therefore reached
the model as mojibake, and once the buffer held correct UTF-8 the input
line rendered as mojibake instead. Windows' C runtime has no ".UTF-8"
locale to switch to, so the conversion has to be explicit.
os_compat.[ch]
* add os_utf8_encode() and os_utf8_decode(), locale-independent, with
range and overlong-form checks
src/tui.c
* input_put_wc(), the command-palette filter and the ask_user buffer use
os_utf8_encode() instead of wcrtomb()
src/textsel.c
* cst_utf8_decode() uses os_utf8_decode() instead of mbrtowc()
This commit is contained in:
@@ -1169,6 +1169,74 @@ void os_proc_kill(os_proc_t *p) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* --- UTF-8 ---------------------------------------------------------------
|
||||
* See the declaration in os_compat.h for why wcrtomb() is not used. */
|
||||
int os_utf8_encode(unsigned long cp, char out[4]) {
|
||||
if (cp < 0x80) {
|
||||
out[0] = (char)cp;
|
||||
return 1;
|
||||
}
|
||||
if (cp < 0x800) {
|
||||
out[0] = (char)(0xC0 | (cp >> 6));
|
||||
out[1] = (char)(0x80 | (cp & 0x3F));
|
||||
return 2;
|
||||
}
|
||||
if (cp >= 0xD800 && cp <= 0xDFFF)
|
||||
return 0; /* lone surrogate half from a 16-bit wchar_t */
|
||||
if (cp < 0x10000) {
|
||||
out[0] = (char)(0xE0 | (cp >> 12));
|
||||
out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||
out[2] = (char)(0x80 | (cp & 0x3F));
|
||||
return 3;
|
||||
}
|
||||
if (cp <= 0x10FFFF) {
|
||||
out[0] = (char)(0xF0 | (cp >> 18));
|
||||
out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
|
||||
out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||
out[3] = (char)(0x80 | (cp & 0x3F));
|
||||
return 4;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* See the declaration in os_compat.h for why mbrtowc() is not used. */
|
||||
int os_utf8_decode(const char *s, size_t avail, unsigned long *cp) {
|
||||
const unsigned char *p = (const unsigned char *)s;
|
||||
int need, i;
|
||||
unsigned long v;
|
||||
static const unsigned long MINV[4] = {0, 0x80, 0x800, 0x10000};
|
||||
|
||||
if (!s || !cp || avail == 0)
|
||||
return 0;
|
||||
if (p[0] < 0x80) {
|
||||
*cp = p[0];
|
||||
return 1;
|
||||
}
|
||||
if ((p[0] & 0xE0) == 0xC0) {
|
||||
need = 1;
|
||||
v = p[0] & 0x1Fu;
|
||||
} else if ((p[0] & 0xF0) == 0xE0) {
|
||||
need = 2;
|
||||
v = p[0] & 0x0Fu;
|
||||
} else if ((p[0] & 0xF8) == 0xF0) {
|
||||
need = 3;
|
||||
v = p[0] & 0x07u;
|
||||
} else {
|
||||
return 0; /* not a UTF-8 lead byte */
|
||||
}
|
||||
if (avail < (size_t)need + 1)
|
||||
return 0; /* truncated */
|
||||
for (i = 1; i <= need; i++) {
|
||||
if ((p[i] & 0xC0) != 0x80)
|
||||
return 0; /* bad continuation byte */
|
||||
v = (v << 6) | (p[i] & 0x3Fu);
|
||||
}
|
||||
if (v < MINV[need] || (v >= 0xD800 && v <= 0xDFFF) || v > 0x10FFFF)
|
||||
return 0; /* overlong, surrogate, or out of range */
|
||||
*cp = v;
|
||||
return need + 1;
|
||||
}
|
||||
|
||||
/* --- libc gaps on Windows -----------------------------------------------
|
||||
* strcasestr() is a GNU extension and setenv()/unsetenv() are POSIX; glibc
|
||||
* has all three, MinGW-w64 has none of them. src/tui.c needs them (command
|
||||
|
||||
Reference in New Issue
Block a user