/* UTF-8 everywhere: Chinese workspace and file names through the file * tools and the shell, CRLF and BOM files edited by apply_patch, write_file * and edit_file without losing their style, and (Windows) child output in * the OEM code page turned into UTF-8. */ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include "cst_agent.h" #include "cst_util.h" #include "os_compat.h" #include #include #include #include #include "cst_test_util.h" static int nfail; #define EXPECT(cond, msg) \ do { \ if (!(cond)) { \ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ nfail++; \ } \ } while (0) static char g_ws[1100]; /* tool(name, key, value, key, value, ..., NULL) */ static int tool(cst_tool_result_t *r, const char *name, ...) { ctl_tool_call_t c; memset(&c, 0, sizeof(c)); snprintf(c.id, sizeof(c.id), "t"); snprintf(c.name, sizeof(c.name), "%s", name); va_list ap; va_start(ap, name); const char *k; while ((k = va_arg(ap, const char *)) != NULL) { const char *v = va_arg(ap, const char *); snprintf(c.args[c.nargs].key, sizeof(c.args[c.nargs].key), "%s", k); c.args[c.nargs].value = (char *)v; c.args[c.nargs].value_len = strlen(v); c.nargs++; } va_end(ap); memset(r, 0, sizeof(*r)); cst_tool_dispatch(g_ws, &c, r); return r->exit_code; } static char *slurp(const char *rel, size_t *n) { char p[1200]; snprintf(p, sizeof(p), "%s/%s", g_ws, rel); FILE *f = fopen(p, "rb"); if (!f) return NULL; char *b = malloc(1 << 16); *n = b ? fread(b, 1, (1 << 16) - 1, f) : 0; if (b) b[*n] = 0; fclose(f); return b; } static void spit(const char *rel, const char *s) { char p[1200]; snprintf(p, sizeof(p), "%s/%s", g_ws, rel); FILE *f = fopen(p, "wb"); if (f) { fputs(s, f); fclose(f); } } static int file_is(const char *rel, const char *want) { size_t n = 0; char *b = slurp(rel, &n); int ok = b && n == strlen(want) && memcmp(b, want, n) == 0; if (!ok) fprintf(stderr, " %s is \"%s\"\n", rel, b ? b : "(missing)"); free(b); return ok; } static void test_patch_buffer(void) { char *out = NULL, err[256]; size_t n = 0; int nh = 0; const char *crlf = "a\r\nb\r\nc\r\n"; const char *hunk = "@@ -1,3 +1,3 @@\n a\n-b\n+B\n c\n"; EXPECT(cst_patch_apply_buffer(crlf, strlen(crlf), hunk, strlen(hunk), &out, &n, &nh, err, sizeof(err)) == 0, "LF hunk applies to a CRLF file"); EXPECT(out && strcmp(out, "a\r\nB\r\nc\r\n") == 0, "CRLF kept"); free(out); const char *bom = "\xEF\xBB\xBF" "first\nsecond\n"; const char *h2 = "@@ -1,2 +1,2 @@\n-first\n+FIRST\n second\n"; EXPECT(cst_patch_apply_buffer(bom, strlen(bom), h2, strlen(h2), &out, &n, &nh, err, sizeof(err)) == 0, "first line matches past the BOM"); EXPECT(out && strcmp(out, "\xEF\xBB\xBF" "FIRST\nsecond\n") == 0, "BOM kept"); free(out); const char *lf = "x\ny\n"; const char *h3 = "@@ -1,2 +1,3 @@\n x\n+中文\n y\n"; EXPECT(cst_patch_apply_buffer(lf, strlen(lf), h3, strlen(h3), &out, &n, &nh, err, sizeof(err)) == 0 && out && strcmp(out, "x\n中文\ny\n") == 0, "LF file stays LF"); free(out); EXPECT(cst_text_is_crlf("a\r\nb\r\n", 6), "is_crlf"); EXPECT(!cst_text_is_crlf("a\r\nb\n", 5), "mixed is not crlf"); EXPECT(!cst_text_is_crlf("ab", 2), "no newline is not crlf"); } static void test_tools(void) { cst_tool_result_t r; /* Chinese names through write/read/list/glob/grep */ EXPECT(tool(&r, "write_file", "path", "目录/文件.txt", "content", "你好，世界\n第二行\n", NULL) == 0, "write_file with a Chinese path"); cst_tool_result_free(&r); EXPECT(tool(&r, "read_file", "path", "目录/文件.txt", NULL) == 0 && r.out && strstr(r.out, "你好，世界"), "read_file with a Chinese path"); cst_tool_result_free(&r); EXPECT(tool(&r, "list_dir", "path", "目录", NULL) == 0 && r.out && strstr(r.out, "文件.txt"), "list_dir shows the Chinese name"); cst_tool_result_free(&r); EXPECT(tool(&r, "glob", "pattern", "**/*.txt", NULL) == 0 && r.out && strstr(r.out, "目录/文件.txt"), "glob finds the Chinese name"); cst_tool_result_free(&r); EXPECT(tool(&r, "grep", "pattern", "世界", NULL) == 0 && r.out && strstr(r.out, "文件.txt"), "grep matches Chinese text"); cst_tool_result_free(&r); /* CRLF + BOM file */ spit("win.txt", "\xEF\xBB\xBFone\r\ntwo\r\nthree\r\n"); EXPECT(tool(&r, "read_file", "path", "win.txt", NULL) == 0 && r.out && strcmp(r.out, "one\ntwo\nthree\n") == 0, "read_file hides BOM and CR"); cst_tool_result_free(&r); EXPECT(tool(&r, "edit_file", "path", "win.txt", "old_string", "two", "new_string", "二\n2", NULL) == 0, "edit_file single-line match"); cst_tool_result_free(&r); EXPECT(file_is("win.txt", "\xEF\xBB\xBFone\r\n二\r\n2\r\nthree\r\n"), "edit_file inserts CRLF"); EXPECT(tool(&r, "edit_file", "path", "win.txt", "old_string", "one\n二", "new_string", "壹\n贰", NULL) == 0, "edit_file multi-line LF match"); cst_tool_result_free(&r); EXPECT(file_is("win.txt", "\xEF\xBB\xBF壹\r\n贰\r\n2\r\nthree\r\n"), "edit_file keeps CRLF and BOM"); EXPECT(tool(&r, "apply_patch", "patch", "--- a/win.txt\n+++ b/win.txt\n@@ -3,2 +3,2 @@\n 2\n-three\n" "+叁\n", NULL) == 0, "apply_patch on a CRLF file"); if (r.exit_code) fprintf(stderr, " apply_patch: %s\n", r.out ? r.out : ""); cst_tool_result_free(&r); EXPECT(file_is("win.txt", "\xEF\xBB\xBF壹\r\n贰\r\n2\r\n叁\r\n"), "apply_patch keeps CRLF and BOM"); EXPECT(tool(&r, "write_file", "path", "win.txt", "content", "new\ntext\n", NULL) == 0, "write_file over a CRLF file"); cst_tool_result_free(&r); EXPECT(file_is("win.txt", "\xEF\xBB\xBFnew\r\ntext\r\n"), "write_file keeps CRLF and BOM"); EXPECT(tool(&r, "write_file", "path", "new.txt", "content", "lf\n", NULL) == 0 && file_is("new.txt", "lf\n"), "new files are LF without BOM"); cst_tool_result_free(&r); /* the shell sees and prints UTF-8 */ EXPECT(tool(&r, "run_shell", "cmd", "cat 目录/文件.txt && echo 完成", NULL) == 0 && r.out && strstr(r.out, "你好，世界") && strstr(r.out, "完成"), "run_shell with Chinese arguments and output"); if (!r.out || !strstr(r.out, "完成")) fprintf(stderr, " run_shell: %s\n", r.out ? r.out : ""); cst_tool_result_free(&r); } #ifdef _WIN32 static void test_oem_output(void) { /* "中文" in GBK */ const char gbk[] = "\xD6\xD0\xCE\xC4\r\n"; size_t n = 0; char *u = os_output_to_utf8(gbk, sizeof(gbk) - 1, &n); wchar_t b[8] = L"0"; GetLocaleInfoW(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTCODEPAGE, b, 8); unsigned oem = (unsigned)wcstoul(b, NULL, 10); if (oem == 936) EXPECT(u && strstr(u, "中文"), "GBK output decoded on a 936 system"); else EXPECT(u != NULL, "non-UTF-8 output is decoded"); free(u); EXPECT(os_output_to_utf8("中文", strlen("中文"), &n) == NULL, "UTF-8 output left alone"); printf("test_win_utf8: ACP %u, OEM %u, system OEM %u, console %u\n", GetACP(), GetOEMCP(), oem, GetConsoleOutputCP()); EXPECT(GetACP() == CP_UTF8, "manifest makes UTF-8 the process code page"); } #endif int main(void) { os_console_init(); char base[1024]; if (test_tmpdir(base, sizeof(base), "cst-utf8") != 0) { perror("tmpdir"); return 1; } snprintf(g_ws, sizeof(g_ws), "%s/项目 空格", base); test_mkdir(g_ws, 0755); test_patch_buffer(); test_tools(); #ifdef _WIN32 test_oem_output(); #endif test_rmrf(base); if (nfail) { fprintf(stderr, "test_win_utf8: %d failure(s)\n", nfail); return 1; } puts("test_win_utf8: OK"); return 0; }