Windows: encode and decode text as UTF-8 instead of trusting the locale

setlocale(LC_ALL, "") maps to the ANSI code page on Windows -- 936 (GBK)
on a Chinese system -- so wcrtomb() produces GBK where the protocol wants
UTF-8, and mbrtowc() reads UTF-8 as GBK. Typed Chinese therefore reached
the model as mojibake, and once the buffer held correct UTF-8 the input
line rendered as mojibake instead. Windows' C runtime has no ".UTF-8"
locale to switch to, so the conversion has to be explicit.

os_compat.[ch]
  * add os_utf8_encode() and os_utf8_decode(), locale-independent, with
    range and overlong-form checks
src/tui.c
  * input_put_wc(), the command-palette filter and the ask_user buffer use
    os_utf8_encode() instead of wcrtomb()
src/textsel.c
  * cst_utf8_decode() uses os_utf8_decode() instead of mbrtowc()
This commit is contained in:
XYZ
2026-10-06 17:56:55 +08:00
parent 3db33eddac
commit 929f7125b8
4 changed files with 121 additions and 30 deletions
+68
View File
@@ -1169,6 +1169,74 @@ void os_proc_kill(os_proc_t *p) {
}
#endif
/* --- UTF-8 ---------------------------------------------------------------
* See the declaration in os_compat.h for why wcrtomb() is not used. */
int os_utf8_encode(unsigned long cp, char out[4]) {
if (cp < 0x80) {
out[0] = (char)cp;
return 1;
}
if (cp < 0x800) {
out[0] = (char)(0xC0 | (cp >> 6));
out[1] = (char)(0x80 | (cp & 0x3F));
return 2;
}
if (cp >= 0xD800 && cp <= 0xDFFF)
return 0; /* lone surrogate half from a 16-bit wchar_t */
if (cp < 0x10000) {
out[0] = (char)(0xE0 | (cp >> 12));
out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
out[2] = (char)(0x80 | (cp & 0x3F));
return 3;
}
if (cp <= 0x10FFFF) {
out[0] = (char)(0xF0 | (cp >> 18));
out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
out[3] = (char)(0x80 | (cp & 0x3F));
return 4;
}
return 0;
}
/* See the declaration in os_compat.h for why mbrtowc() is not used. */
int os_utf8_decode(const char *s, size_t avail, unsigned long *cp) {
const unsigned char *p = (const unsigned char *)s;
int need, i;
unsigned long v;
static const unsigned long MINV[4] = {0, 0x80, 0x800, 0x10000};
if (!s || !cp || avail == 0)
return 0;
if (p[0] < 0x80) {
*cp = p[0];
return 1;
}
if ((p[0] & 0xE0) == 0xC0) {
need = 1;
v = p[0] & 0x1Fu;
} else if ((p[0] & 0xF0) == 0xE0) {
need = 2;
v = p[0] & 0x0Fu;
} else if ((p[0] & 0xF8) == 0xF0) {
need = 3;
v = p[0] & 0x07u;
} else {
return 0; /* not a UTF-8 lead byte */
}
if (avail < (size_t)need + 1)
return 0; /* truncated */
for (i = 1; i <= need; i++) {
if ((p[i] & 0xC0) != 0x80)
return 0; /* bad continuation byte */
v = (v << 6) | (p[i] & 0x3Fu);
}
if (v < MINV[need] || (v >= 0xD800 && v <= 0xDFFF) || v > 0x10FFFF)
return 0; /* overlong, surrogate, or out of range */
*cp = v;
return need + 1;
}
/* --- libc gaps on Windows -----------------------------------------------
* strcasestr() is a GNU extension and setenv()/unsetenv() are POSIX; glibc
* has all three, MinGW-w64 has none of them. src/tui.c needs them (command