From 929f7125b87e19b674fecc7ed75a7c6dd1bf7031 Mon Sep 17 00:00:00 2001 From: XYZ <4+xyz@noreply.git.whcself.xyz> Date: Tue, 6 Oct 2026 17:56:55 +0800 Subject: [PATCH] Windows: encode and decode text as UTF-8 instead of trusting the locale setlocale(LC_ALL, "") maps to the ANSI code page on Windows -- 936 (GBK) on a Chinese system -- so wcrtomb() produces GBK where the protocol wants UTF-8, and mbrtowc() reads UTF-8 as GBK. Typed Chinese therefore reached the model as mojibake, and once the buffer held correct UTF-8 the input line rendered as mojibake instead. Windows' C runtime has no ".UTF-8" locale to switch to, so the conversion has to be explicit. os_compat.[ch] * add os_utf8_encode() and os_utf8_decode(), locale-independent, with range and overlong-form checks src/tui.c * input_put_wc(), the command-palette filter and the ask_user buffer use os_utf8_encode() instead of wcrtomb() src/textsel.c * cst_utf8_decode() uses os_utf8_decode() instead of mbrtowc() --- include/os_compat.h | 23 +++++++++++++++ src/os_compat.c | 68 +++++++++++++++++++++++++++++++++++++++++++++ src/textsel.c | 18 ++++++++---- src/tui.c | 42 ++++++++++++---------------- 4 files changed, 121 insertions(+), 30 deletions(-) diff --git a/include/os_compat.h b/include/os_compat.h index 0ef9a57..4d72c7f 100644 --- a/include/os_compat.h +++ b/include/os_compat.h @@ -111,6 +111,29 @@ void os_localtime(const time_t *t, struct tm *out); int os_getpid(void); int os_chmod_private(const char *path); /* best-effort 0600; no-op/ok on Win */ +/* --- UTF-8 -------------------------------------------------------------- + * Encode one code point as UTF-8, independent of the C locale. + * + * wcrtomb() follows setlocale(). On Linux the environment normally selects a + * UTF-8 locale, but on Windows the C runtime maps setlocale(LC_ALL, "") to the + * ANSI code page -- 936 (GBK) on a Chinese system -- so wcrtomb() hands back + * GBK bytes where the protocol requires UTF-8, and the model receives + * mojibake. Windows' C runtime has no ".UTF-8" locale to switch to. + * + * Returns the number of bytes written (1..4), or 0 if the code point cannot be + * encoded. Surrogate halves (as they arrive on 16-bit wchar_t platforms for + * non-BMP code points) return 0 and are skipped. */ +int os_utf8_encode(unsigned long cp, char out[4]); + +/* Decode one UTF-8 sequence from s (at most avail bytes). + * Returns the number of bytes consumed (1..4) and stores the code point in + * *cp, or 0 if the bytes are not a valid, complete, non-overlong sequence. + * + * The counterpart of the encoder above, and needed for the same reason: + * mbrtowc() also follows setlocale(), so on Windows it would decode UTF-8 + * text as GBK and the screen would show mojibake. */ +int os_utf8_decode(const char *s, size_t avail, unsigned long *cp); + /* --- libc gaps on Windows ------------------------------------------------ * MinGW-w64 provides neither strcasestr() (a GNU extension) nor * setenv()/unsetenv() (POSIX): they are simply absent from its headers. diff --git a/src/os_compat.c b/src/os_compat.c index 6b941ae..a579117 100644 --- a/src/os_compat.c +++ b/src/os_compat.c @@ -1169,6 +1169,74 @@ void os_proc_kill(os_proc_t *p) { } #endif +/* --- UTF-8 --------------------------------------------------------------- + * See the declaration in os_compat.h for why wcrtomb() is not used. */ +int os_utf8_encode(unsigned long cp, char out[4]) { + if (cp < 0x80) { + out[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + if (cp >= 0xD800 && cp <= 0xDFFF) + return 0; /* lone surrogate half from a 16-bit wchar_t */ + if (cp < 0x10000) { + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + if (cp <= 0x10FFFF) { + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; + } + return 0; +} + +/* See the declaration in os_compat.h for why mbrtowc() is not used. */ +int os_utf8_decode(const char *s, size_t avail, unsigned long *cp) { + const unsigned char *p = (const unsigned char *)s; + int need, i; + unsigned long v; + static const unsigned long MINV[4] = {0, 0x80, 0x800, 0x10000}; + + if (!s || !cp || avail == 0) + return 0; + if (p[0] < 0x80) { + *cp = p[0]; + return 1; + } + if ((p[0] & 0xE0) == 0xC0) { + need = 1; + v = p[0] & 0x1Fu; + } else if ((p[0] & 0xF0) == 0xE0) { + need = 2; + v = p[0] & 0x0Fu; + } else if ((p[0] & 0xF8) == 0xF0) { + need = 3; + v = p[0] & 0x07u; + } else { + return 0; /* not a UTF-8 lead byte */ + } + if (avail < (size_t)need + 1) + return 0; /* truncated */ + for (i = 1; i <= need; i++) { + if ((p[i] & 0xC0) != 0x80) + return 0; /* bad continuation byte */ + v = (v << 6) | (p[i] & 0x3Fu); + } + if (v < MINV[need] || (v >= 0xD800 && v <= 0xDFFF) || v > 0x10FFFF) + return 0; /* overlong, surrogate, or out of range */ + *cp = v; + return need + 1; +} + /* --- libc gaps on Windows ----------------------------------------------- * strcasestr() is a GNU extension and setenv()/unsetenv() are POSIX; glibc * has all three, MinGW-w64 has none of them. src/tui.c needs them (command diff --git a/src/textsel.c b/src/textsel.c index 0be8583..18bae3b 100644 --- a/src/textsel.c +++ b/src/textsel.c @@ -3,23 +3,29 @@ #endif #include "cst_textsel.h" +#include "os_compat.h" #include #include #include int cst_utf8_decode(const char *s, size_t avail, wchar_t *wc, size_t *eaten) { + unsigned long cp = 0; + int n; if (avail == 0) return -1; - mbstate_t st; - memset(&st, 0, sizeof(st)); - size_t n = mbrtowc(wc, s, avail, &st); - if (n == (size_t)-1 || n == (size_t)-2 || n == 0) { - *wc = (unsigned char)s[0]; + /* os_utf8_decode(), not mbrtowc(): the latter follows setlocale(), which on + * Windows means the ANSI code page, so UTF-8 text would be decoded as GBK + * and rendered as mojibake. */ + n = os_utf8_decode(s, avail, &cp); + if (n <= 0) { + /* Not a valid sequence: fall back to one raw byte, as before. */ + *wc = (wchar_t)(unsigned char)s[0]; *eaten = 1; return 0; } - *eaten = n; + *wc = (wchar_t)cp; + *eaten = (size_t)n; return 0; } diff --git a/src/tui.c b/src/tui.c index 2a1866a..c58c4a6 100644 --- a/src/tui.c +++ b/src/tui.c @@ -2966,12 +2966,10 @@ static void input_clear(tui_t *t) { input_set(t, ""); } static void input_put_wc(tui_t *t, wchar_t wc) { char mb[8]; - mbstate_t st; - memset(&st, 0, sizeof(st)); - size_t n = wcrtomb(mb, wc, &st); - if (n == (size_t)-1 || n == 0) + int n = os_utf8_encode((unsigned long)wc, mb); + if (n <= 0) return; - input_insert(t, mb, n); + input_insert(t, mb, (size_t)n); } static void input_put_byte(tui_t *t, char ch) { input_insert(t, &ch, 1); } @@ -3600,16 +3598,14 @@ static int picker_run(tui_t *t, const char *title, char items[][128], int n, continue; } if (iswprint(wch)) { - char mb[8]; - mbstate_t st; - memset(&st, 0, sizeof(st)); - size_t k = wcrtomb(mb, (wchar_t)wch, &st); - if (k != (size_t)-1 && qlen + k < sizeof(query)) { - memcpy(query + qlen, mb, k); - qlen += k; - query[qlen] = 0; - sel = top = 0; - } + char mb[8]; + int k = os_utf8_encode((unsigned long)(wchar_t)wch, mb); + if (k > 0 && qlen + (size_t)k < sizeof(query)) { + memcpy(query + qlen, mb, (size_t)k); + qlen += (size_t)k; + query[qlen] = 0; + sel = top = 0; + } } } @@ -6143,15 +6139,13 @@ static void run_ask_modal(tui_t *t) { break; } if (iswprint(wch)) { - char mb[8]; - mbstate_t st; - memset(&st, 0, sizeof(st)); - size_t k = wcrtomb(mb, (wchar_t)wch, &st); - size_t n = strlen(t->ask_buf); - if (k != (size_t)-1 && n + k + 1 < sizeof(t->ask_buf)) { - memcpy(t->ask_buf + n, mb, k); - t->ask_buf[n + k] = 0; - } + char mb[8]; + int k = os_utf8_encode((unsigned long)(wchar_t)wch, mb); + size_t n = strlen(t->ask_buf); + if (k > 0 && n + (size_t)k + 1 < sizeof(t->ask_buf)) { + memcpy(t->ask_buf + n, mb, (size_t)k); + t->ask_buf[n + k] = 0; + } } } curs_set(1);