diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8517edc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.cmd text eol=crlf +*.bat text eol=crlf diff --git a/.gitignore b/.gitignore index c561d09..3f78207 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ third_party/busybox/busybox third_party/busybox/busybox.exe third_party/curl/ third_party/PDCurses/ +third_party/ncurses-*/ +third_party/ncurses-*.tar.gz third_party/busybox-w32/ third_party/busybox-src/ third_party/prefix-mingw/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d4a218..abdb70e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,32 @@ option(CST_BUILD_TUI "Build ncurses/PDCurses TUI" ON) option(CST_NATIVE_ARCH "Pass -march=native to sashidown" OFF) option(CST_BUNDLE_BUSYBOX "Install placeholder/scripts for busybox beside binary" ON) +if(WIN32) + enable_language(RC) + # Cross builds link libgcc/winpthreads/curl statically; MSYS2's curl is a DLL. + if(CMAKE_CROSSCOMPILING) + set(_CST_STATIC_DEFAULT ON) + else() + set(_CST_STATIC_DEFAULT OFF) + endif() + option(CST_WIN_STATIC "Link Windows executables with -static" ${_CST_STATIC_DEFAULT}) +endif() + +# Windows: UTF-8 manifest (activeCodePage, longPathAware) and version resource +# on every executable, so tests run with the same process code page as the app. +function(cst_win_exe t) + if(NOT WIN32) + return() + endif() + target_sources(${t} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/res/windows/cst-agent.rc") + target_include_directories(${t} PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/res/windows") + target_compile_definitions(${t} PRIVATE _WIN32_WINNT=0x0A00) + if(CST_WIN_STATIC) + target_link_options(${t} PRIVATE -static) + endif() +endfunction() + set(SD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/sashidown") if(CST_NATIVE_ARCH) set(_SD_MAKE_EXTRA CST_NATIVE_ARCH=1) @@ -15,6 +41,9 @@ else() set(_SD_MAKE_EXTRA "") endif() +# sashidown has a plain Makefile; CMAKE_MAKE_PROGRAM may be ninja. +find_program(CST_MAKE NAMES make gmake mingw32-make REQUIRED) + # Separate archive for MinGW so Linux host builds keep libsashidown.a if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(SASHIDOWN_A "${SD_DIR}/libsashidown-mingw.a") @@ -26,19 +55,19 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(_MINGW_RANLIB "${CMAKE_RANLIB}") add_custom_command( OUTPUT "${SASHIDOWN_A}" - COMMAND ${CMAKE_MAKE_PROGRAM} -C "${SD_DIR}" clean - COMMAND ${CMAKE_MAKE_PROGRAM} -C "${SD_DIR}" libsashidown.a + COMMAND ${CST_MAKE} -C "${SD_DIR}" clean + COMMAND ${CST_MAKE} -C "${SD_DIR}" libsashidown.a CC=${CMAKE_C_COMPILER} AR=${_MINGW_AR} RANLIB=${_MINGW_RANLIB} COMMAND ${CMAKE_COMMAND} -E copy_if_different "${SD_DIR}/libsashidown.a" "${SASHIDOWN_A}" - COMMAND ${CMAKE_MAKE_PROGRAM} -C "${SD_DIR}" clean + COMMAND ${CST_MAKE} -C "${SD_DIR}" clean COMMENT "Building libsashidown for MinGW" VERBATIM) else() set(SASHIDOWN_A "${SD_DIR}/libsashidown.a") add_custom_command( OUTPUT "${SASHIDOWN_A}" - COMMAND ${CMAKE_MAKE_PROGRAM} -C "${SD_DIR}" libsashidown.a CC=${CMAKE_C_COMPILER} ${_SD_MAKE_EXTRA} + COMMAND ${CST_MAKE} -C "${SD_DIR}" libsashidown.a CC=${CMAKE_C_COMPILER} ${_SD_MAKE_EXTRA} COMMENT "Building libsashidown.a" VERBATIM) endif() @@ -54,6 +83,7 @@ set(CST_CORE_SOURCES src/util.c src/ctl_parse.c src/os_compat.c + src/os_win.c src/sandbox.c src/sandbox_exec.c src/keymap.c @@ -82,6 +112,14 @@ set(CST_CORE_SOURCES src/models.c src/metrics.c ) +if(WIN32) + # MinGW has no : musl's TRE engine, UTF-8 aware (vendor/tre-regex) + include_directories(vendor/tre-regex) + list(APPEND CST_CORE_SOURCES + vendor/tre-regex/regcomp.c vendor/tre-regex/regexec.c + vendor/tre-regex/regerror.c vendor/tre-regex/tre-mem.c) +endif() + set(CST_SOURCES ${CST_CORE_SOURCES} src/main.c) if(CST_BUILD_TUI) @@ -92,6 +130,7 @@ endif() add_executable(cst-agent ${CST_SOURCES}) add_dependencies(cst-agent sashidown) +cst_win_exe(cst-agent) target_include_directories(cst-agent PRIVATE include @@ -113,9 +152,26 @@ if(NOT WIN32) target_link_libraries(cst-agent PRIVATE m) endif() -if(CST_BUILD_TUI) +set(CST_WIN_CURSES "ncurses" CACHE STRING "Windows TUI backend: ncurses or pdcurses") +set_property(CACHE CST_WIN_CURSES PROPERTY STRINGS ncurses pdcurses) +if(CST_BUILD_TUI AND WIN32 AND CST_WIN_CURSES STREQUAL "ncurses") + # ncursesw built by scripts/build_third_party_mingw.sh ncurses (or MSYS2's + # mingw-w64-ucrt-x86_64-ncurses); headers live in include/ncursesw. + find_path(NCURSESW_INCLUDE_DIR ncurses.h PATH_SUFFIXES ncursesw include/ncursesw) + find_library(NCURSESW_LIBRARY NAMES ncursesw) + if(NOT NCURSESW_INCLUDE_DIR OR NOT NCURSESW_LIBRARY) + message(FATAL_ERROR "CST_WIN_CURSES=ncurses needs ncursesw " + "(scripts/build_third_party_mingw.sh ncurses), or -DCST_WIN_CURSES=pdcurses") + endif() + # ncurses.h includes , so the parent is needed too + get_filename_component(NCURSESW_PARENT_DIR ${NCURSESW_INCLUDE_DIR} DIRECTORY) + target_include_directories(cst-agent PRIVATE ${NCURSESW_INCLUDE_DIR} ${NCURSESW_PARENT_DIR}) + target_link_libraries(cst-agent PRIVATE ${NCURSESW_LIBRARY}) + target_compile_definitions(cst-agent PRIVATE CST_USE_NCURSES=1 NCURSES_STATIC=1 NCURSES_WIDECHAR=1) + message(STATUS "TUI: ncursesw (${NCURSESW_LIBRARY})") +elseif(CST_BUILD_TUI) if(WIN32) - # Prefer PDCurses (pdcurses / curses). Users may set PDCurses_ROOT. + # PDCurses (pdcurses / curses). Users may set PDCurses_ROOT. # PATH_SUFFIXES also covers the layout MSYS2's mingw-w64-pdcurses uses, # which puts the headers under include/pdcurses/ rather than include/. find_path(PDCURSES_INCLUDE_DIR curses.h @@ -156,7 +212,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR WIN32) # curl Schannel + networking; winmm is needed by PDCurses (PlaySoundW) target_link_libraries(cst-agent PRIVATE - ws2_32 crypt32 secur32 bcrypt advapi32 winmm) + ws2_32 crypt32 secur32 bcrypt advapi32 user32 gdi32 winmm) endif() # --- tests --- @@ -166,7 +222,11 @@ set(_CST_SMALL_TESTS test_ctl_frontier test_ctl_heredoc test_json_baseline test_markdown) foreach(t ${_CST_SMALL_TESTS}) if(t STREQUAL test_textsel) - add_executable(${t} tests/${t}.c src/textsel.c) + add_executable(${t} tests/${t}.c src/textsel.c src/os_compat.c src/os_win.c + src/util.c) + if(WIN32) + target_link_libraries(${t} PRIVATE advapi32 user32) + endif() elseif(t STREQUAL test_markdown) add_executable(${t} tests/${t}.c) target_include_directories(${t} PRIVATE "${SD_DIR}/include") @@ -186,6 +246,7 @@ foreach(t ${_CST_SMALL_TESTS}) if(NOT WIN32) target_link_libraries(${t} PRIVATE m) endif() + cst_win_exe(${t}) add_test(NAME ${t} COMMAND ${t} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") endforeach() @@ -194,7 +255,8 @@ foreach(t test_llm_body test_sse_decode test_apply_patch test_session_roundtrip test_shell test_diff test_tools_fs test_registry test_undo test_config test_models test_sandbox test_keymap test_shell_compact test_skills - test_plugins test_mcp test_subagent) + test_plugins test_mcp test_subagent test_lowil + test_win_utf8 test_os_dir) add_executable(${t} tests/${t}.c ${CST_CORE_SOURCES}) target_include_directories(${t} PRIVATE include "${CJSON_DIR}") target_compile_definitions(${t} PRIVATE _GNU_SOURCE) @@ -202,8 +264,9 @@ foreach(t test_llm_body test_sse_decode test_apply_patch test_session_roundtrip if(NOT WIN32) target_link_libraries(${t} PRIVATE m) else() - target_link_libraries(${t} PRIVATE ws2_32 crypt32 secur32 bcrypt advapi32) + target_link_libraries(${t} PRIVATE ws2_32 crypt32 secur32 bcrypt advapi32 user32) endif() + cst_win_exe(${t}) add_test(NAME ${t} COMMAND ${t} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") endforeach() diff --git a/Makefile b/Makefile index c9a2cdb..43bd510 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ CORE_SRC = \ src/util.c \ src/ctl_parse.c \ src/os_compat.c \ + src/os_win.c \ src/sandbox.c \ src/sandbox_exec.c \ src/keymap.c \ @@ -110,7 +111,10 @@ TESTS = \ tests/test_skills \ tests/test_plugins \ tests/test_mcp \ - tests/test_subagent + tests/test_subagent \ + tests/test_lowil \ + tests/test_win_utf8 \ + tests/test_os_dir TEST_LIBS = -lm $(CURL_LIBS) $(PTHREAD_LIBS) @@ -163,8 +167,8 @@ tests/test_replay_real: tests/test_replay_real.c src/ctl_parse.c src/json_baseli tests/test_mm_tool: tests/test_mm_tool.c src/mm_tool.c include/mm_tool.h $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ tests/test_mm_tool.c src/mm_tool.c -tests/test_textsel: tests/test_textsel.c src/textsel.c include/cst_textsel.h - $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ tests/test_textsel.c src/textsel.c +tests/test_textsel: tests/test_textsel.c src/textsel.c src/os_compat.c src/os_win.c src/util.c include/cst_textsel.h + $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ tests/test_textsel.c src/textsel.c src/os_compat.c src/os_win.c src/util.c tests/test_markdown: tests/test_markdown.c $(SD_DIR)/libsashidown.a $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ tests/test_markdown.c $(LDFLAGS) -lsashidown -lm diff --git a/README.md b/README.md index ea7c125..b766e4a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,15 @@ cst-agent --version cd ~/my-project && cst-agent ``` +Windows 10 1903+(PowerShell,不需要管理员): + +```powershell +irm https://git.whcself.xyz/WHC/CST-agent/raw/branch/main/scripts/install_windows.ps1 | iex +# 新开终端后:cd 到项目目录运行 cst-agent;升级用 cst-agent-update +``` + +详见 [docs/README-windows.md](docs/README-windows.md)。 + 一键脚本可以重复运行:已有仓库会 `git pull --ff-only`,代码有更新时干净重编,然后重新安装; 同时装一个 `cst-agent-update` 命令(等同重跑脚本)。NixOS 上会自动进 `nix-shell` 编译。 @@ -39,8 +48,9 @@ cd ~/my-project && cst-agent |---|---| | Linux | [`scripts/bootstrap_linux.sh`](scripts/bootstrap_linux.sh) | | Termux | [`scripts/bootstrap_termux.sh`](scripts/bootstrap_termux.sh) | +| Windows 安装 / 升级 | [`scripts/install_windows.ps1`](scripts/install_windows.ps1)(从 Release 安装),之后 `cst-agent-update` | | Windows 交叉构建(在 Linux 上) | [`scripts/bootstrap_windows.sh`](scripts/bootstrap_windows.sh) → `dist/windows-x86_64/` | -| Windows 运行 | 拷贝 dist 后 [`scripts/run_windows.bat`](scripts/run_windows.bat) | +| Windows 原生构建(MSYS2 UCRT64) | [`scripts/bootstrap_msys2.sh`](scripts/bootstrap_msys2.sh) → `dist/windows-x86_64/` | ### 迷你验收工程(论文案例) @@ -97,14 +107,11 @@ cmake --build build-clang # Headless only(无 ncurses) cmake -B build-hd -DCST_BUILD_TUI=OFF && cmake --build build-hd -# Windows MinGW 交叉(需安装 mingw-w64) -cmake -B build-win -DCMAKE_TOOLCHAIN_FILE=cmake/mingw-w64-x86_64.cmake \ - -DCST_BUILD_TUI=OFF -cmake --build build-win -# TUI:安装 PDCurses 后 -DCST_BUILD_TUI=ON -DPDCurses_ROOT=... +# Windows:用一键脚本(见下方“Windows 发行包”),会先编译静态依赖 +./scripts/bootstrap_windows.sh ``` -依赖:`gcc` 或 `clang`、`libcurl`、Linux TUI 需 `ncursesw`;Windows TUI 需 **PDCurses**。 +依赖:`gcc` 或 `clang`、`libcurl`、TUI 需 `ncursesw`(Windows 版用打过补丁的 ncurses 6.6,脚本自动编译)。 Sashidown(`vendor/sashidown`)与 cJSON(`vendor/cjson`,MIT)已随仓库提供。 API Key:TUI 里输入并按 provider 保存到配置目录的 `credentials`(权限 0600);`/key` 更换,`--ask-key` 在命令行输入。 `--api-key` 只用于本次进程、不落盘;`DEEPSEEK_API_KEY` 等环境变量和 `CST_API_KEY` 仅作为没有保存 key 时的兜底(方便 CI)。 @@ -117,7 +124,7 @@ API Key:TUI 里输入并按 provider 保存到配置目录的 `credentials`( | WSL2 | x86_64 | 同 Linux | yes | ncursesw | `/bin/sh`(工作区建议放在 Linux 文件系统内) | | macOS | arm64 / x86_64 | Apple Clang | yes | ncurses(Homebrew) | `/bin/sh` | | Android Termux | aarch64 / … | Clang (Termux) | yes | ncurses(Termux 包) | `$PREFIX/bin/sh`;可旁路 busybox | -| Windows | x86_64 / aarch64 | MinGW GCC/Clang | yes | PDCurses | **捆绑 busybox-w32** | +| Windows 10 1903+ | x86_64 | MinGW-w64 GCC(UCRT) | yes | ncurses 6.6(Windows 控制台) | **捆绑 busybox-w32** | ### Termux @@ -140,17 +147,14 @@ make && make install # 装到 Termux 的 $PREFIX/bin,之后任意目录 ### Windows 发行包 -见 [docs/README-windows.md](docs/README-windows.md)。交叉构建: +见 [docs/README-windows.md](docs/README-windows.md)(安装、编码、沙箱、构建、已知限制)。 ```bash -./scripts/build_third_party_mingw.sh all # curl+PDCurses+busybox-w32 -# 源码:https://github.com/rmyorston/busybox-w32.git -# https://github.com/mirror/busybox.git (Unix) -cmake -B build-win-tui -DCMAKE_TOOLCHAIN_FILE=cmake/mingw-w64-x86_64.cmake \ - -DCST_BUILD_TUI=ON -DCMAKE_PREFIX_PATH=$PWD/third_party/prefix-mingw +./scripts/bootstrap_windows.sh # Linux 交叉编译;Windows 上在 MSYS2 UCRT64 里用 scripts/bootstrap_msys2.sh ``` -产物示例:`dist/windows-x86_64/`(含 `cst-agent.exe` + `busybox.exe` + `libwinpthread-1.dll`)。 +产物:`dist/windows-x86_64/`(静态链接的 `cst-agent.exe` + `busybox.exe`,无其他 DLL)和 +`dist/cst-agent-VERSION-windows-x86_64.zip`;`GITEA_TOKEN=... scripts/release_windows.sh` 上传到 Release。 环境变量:`CST_SHELL`、`CST_EXE_DIR`、`CST_AUTO_APPROVE` 等。 @@ -275,8 +279,11 @@ TUI 中 `/tools off NAME`、`/tools on NAME` 可临时开关单个工具。 读、执行和已打开文件的写入不检查。它防的是误操作而不是恶意程序:检查之后内核会再读一次路径,多线程程序可以趁机换掉。 比 shell 活得久的后台进程:桌面发行版开着 Yama(`ptrace_scope` ≥ 1)时它们的受检写入会被拒;命令结束后一律失败。 - macOS:`sandbox_init` profile,同样只放行工作区和临时目录写入,禁网。 - - Windows、更老的内核:没有沙箱,自动回落到规则判断(界面显示 `auto · rules`)。 - - `CST_SANDBOX=off` 强制关闭沙箱;`CST_SANDBOX=landlock` / `seccomp` 只用指定的那种。 + - Windows 10 1903+:Low 完整性级别的受限令牌 + Job Object(界面显示 `auto · lowil`)。第一次在工作区运行时 + 给它打上 Low 标签,之后子进程只能写工作区和沙箱临时目录,读不到凭据文件;不限制联网,所以联网命令仍会询问。 + `/sandbox` 查看、`/sandbox reset-labels` 撤销标签,详见 [docs/README-windows.md](docs/README-windows.md)。 + - 更老的内核:没有沙箱,自动回落到规则判断(界面显示 `auto · rules`)。 + - `CST_SANDBOX=off` 强制关闭沙箱;`CST_SANDBOX=landlock` / `seccomp` / `lowil` 只用指定的那种。 - 沙箱里的命令如果像是被拦了(`Permission denied` / `权限不够` / `Read-only file system` 等),会弹窗问 “Rerun outside the sandbox?”:`y` 在沙箱外重跑一次 · `p` 保存命令前缀规则并重跑 · `n` 保留沙箱结果(模型会收到说明)。 - **只读白名单**(无沙箱时 `auto` 用):`ls cat head tail wc grep rg find pwd echo which file stat du tree` diff --git a/docs/README-windows.md b/docs/README-windows.md index b2099d4..7e8ebc4 100644 --- a/docs/README-windows.md +++ b/docs/README-windows.md @@ -1,82 +1,177 @@ # CST-Agent on Windows -## Release layout +原生 Windows 版:一个静态链接的 `cst-agent.exe`(UCRT,内嵌 UTF-8 清单)加一个 +`busybox.exe`(busybox-w32 Unicode 版)作为 shell,不依赖其他 DLL,也不需要管理员权限。 -``` -cst-agent.exe -busybox.exe # built from busybox-w32 -libwinpthread-1.dll # MinGW runtime (required) -LICENSE.busybox -README-windows.md +## 安装 + +在 PowerShell 里(Windows PowerShell 5.1 或 PowerShell 7 都行): + +```powershell +irm https://git.whcself.xyz/WHC/CST-agent/raw/branch/main/scripts/install_windows.ps1 | iex ``` -Prebuilt tree from Linux cross-compile: `agent/dist/windows-x86_64/`. - -## Build (MinGW-w64 cross from Linux) - -```bash -cd agent -# 1) third-party: curl (Schannel) + PDCurses + busybox-w32 -./scripts/build_third_party_mingw.sh all -# sources: -# https://github.com/rmyorston/busybox-w32.git -# https://github.com/curl/curl.git -# https://github.com/wmcbrine/PDCurses.git -# Unix busybox (optional, Linux/Termux bundle): -# ./scripts/build_third_party_mingw.sh busybox-unix -# https://github.com/mirror/busybox.git - -# 2) headless -cmake -B build-win -DCMAKE_TOOLCHAIN_FILE=cmake/mingw-w64-x86_64.cmake \ - -DCST_BUILD_TUI=OFF -DCMAKE_PREFIX_PATH=$PWD/third_party/prefix-mingw -cmake --build build-win - -# 3) TUI (PDCurses) -cmake -B build-win-tui -DCMAKE_TOOLCHAIN_FILE=cmake/mingw-w64-x86_64.cmake \ - -DCST_BUILD_TUI=ON -DCMAKE_PREFIX_PATH=$PWD/third_party/prefix-mingw \ - -DPDCurses_ROOT=$PWD/third_party/prefix-mingw -cmake --build build-win-tui -``` - -Copy `build-win-tui/cst-agent.exe`, `third_party/busybox/busybox.exe`, and -`libwinpthread-1.dll` into one folder. - -## Build natively on Windows (MSYS2) - -The CMake build also works directly on Windows with MSYS2's mingw-w64 -toolchain -- no separate Linux host needed: - -```bash -# in an MSYS2 MINGW64 shell -pacman -S --needed --noconfirm \ - mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-make \ - mingw-w64-x86_64-curl mingw-w64-x86_64-pdcurses - -cd agent -cmake -B build-win -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release -cmake --build build-win --target cst-agent -j"$(nproc)" -``` - -`CST_BUILD_TUI=ON` (the default) picks up MSYS2's PDCurses automatically, -including its `include/pdcurses/` layout; set `-DPDCurses_ROOT=...` only for -a custom PDCurses. - -Two differences from the cross-build release layout above: - -- **Extra DLLs.** MSYS2's curl package is the shared OpenSSL build, so the - executable needs its whole dependency chain beside it. Collect it with - `ldd build-win/cst-agent.exe | grep mingw64`. -- **Only build the `cst-agent` target.** Some sources under `tests/` are - POSIX-only (two-argument `mkdir`, `setenv`, `arpa/inet.h`, `symlink`) and - do not compile with MinGW. The main program is unaffected. - -## Run +脚本从 Gitea Release 下载 `cst-agent-VERSION-windows-x86_64.zip`,校验 `.sha256`, +装到 `%LOCALAPPDATA%\Programs\cst-agent`,并把这个目录加进**用户** PATH。装完新开一个终端: ```bat -set CST_API_KEY=... -cst-agent.exe --headless --workspace . --prompt "echo hi" -cst-agent.exe --workspace . +cd C:\code\my-project +cst-agent ``` -Shell: bundled `busybox.exe sh -c ...` unless `CST_SHELL` is set. -Credentials: `%APPDATA%\cst-agent\credentials` (or console prompt). +首次启动会让你选 provider 并输入 API key(遮罩输入,按 provider 保存)。 + +以后升级: + +```bat +cst-agent-update :: 最新版 +cst-agent-update -Version 0.6.0 :: 指定版本 +``` + +正在运行的 `cst-agent.exe` 不会挡住升级:旧文件会先改名为 `.old`,下次升级时删掉。 + +安装脚本的参数(下载后用 `powershell -ExecutionPolicy Bypass -File install_windows.ps1 ...` 运行): + +| 参数 | 环境变量 | 作用 | +|---|---|---| +| `-Version 0.6.0` | `CST_VERSION` | 指定版本,默认最新 | +| `-Prefix D:\tools\cst` | `CST_PREFIX` | 安装目录 | +| `-Zip FILE` | | 用本地 zip,不联网 | +| `-NoPath` | | 不改用户 PATH | +| | `CST_GITEA` / `CST_REPO` | 换 Gitea 服务器或仓库(默认 `WHC/CST-agent`) | + +不想用脚本的话,把 zip 解压到任意目录直接运行 `cst-agent.exe` 也可以。 + +## 系统要求 + +- Windows 10 1903(build 18362)及以上、Windows 11、Windows Server 2022 / 2025。 + 更老的系统不认清单里的 `activeCodePage=UTF-8`,安装脚本会直接拒绝。 +- x86_64。Windows 10 1903 起系统自带 UCRT,无需另装运行库。 +- 终端:Windows Terminal 或传统控制台(conhost)都行。 + +## 发行包内容 + +``` +windows-x86_64/ + cst-agent.exe # 静态链接:curl(Schannel)、ncurses 6.6、winpthreads + busybox.exe # busybox-w32 Unicode 版,run_shell 用它的 sh + LICENSE.busybox + README.md # 本文件 + VERSION + cst-agent-update.cmd # 调用旁边的 install_windows.ps1 升级本目录 + install_windows.ps1 +``` + +## 文件位置 + +| 内容 | 位置 | +|---|---| +| API key(每个 provider 一个) | `%APPDATA%\cst-agent\credentials`,只有当前用户可读,并带“禁止向上读”标签(沙箱内读不到) | +| 上次的 provider、模型、推理强度、确认模式 | `%APPDATA%\cst-agent\settings.json` | +| 沙箱打过标签的目录 | `%APPDATA%\cst-agent\sandbox-labels` | +| 会话、撤销快照、导出 | `<工作区>\.agent\` | + +设置了绝对路径的 `XDG_CONFIG_HOME` 时用 `%XDG_CONFIG_HOME%\cst-agent`。 + +## 编码:全部 UTF-8 + +- exe 内嵌 `activeCodePage=UTF-8` 清单:进程的 ANSI 代码页就是 65001,命令行参数、 + 环境变量、文件名都以 UTF-8 进出;内部所有路径在系统调用边界转成 UTF-16(`*W` API)。 +- 控制台输入输出代码页在启动时设为 65001,退出时恢复。 +- 文件按字节读写,不做换行转换。`read_file` 给模型看的是去掉 `\r` 的文本;`edit_file`、 + `apply_patch`、`write_file` 会保持原文件的 CRLF 换行和 UTF-8 BOM。 +- 子进程输出:先按 UTF-8 解码;不是合法 UTF-8 时依次尝试控制台代码页、系统 OEM / ANSI + 代码页(比如中文系统的 936),转成 UTF-8 再交给模型。 +- 中文用户名、中文工作区路径(`C:\Users\测试\项目`)都已在中英文系统上测过。 + +## 终端界面(TUI) + +用的是打过补丁的 ncurses 6.6 Windows 控制台驱动(补丁在 +`scripts/patches/ncurses-6.6-win32-console.patch`): + +- 中文按两列宽显示,光标位置正确;粘贴多行不会直接发送。 +- 鼠标:点击、拖选复制(写入 Windows 剪贴板)、滚轮滚动;`/mouse` 关闭。 +- `/copy` 和拖选复制走 Win32 剪贴板 API,不依赖终端的 OSC 52。 +- 控制台是点阵字体时(显示不了中文,会变成 `?`),启动时自动换成同字号的 Consolas + (中文系统换成新宋体),退出时换回去。 +- 传统控制台不做字形回退:当前字体缺 logo 用的方块字符时(比如中文系统默认的新宋体), + 首页改用文字 logo。 + +## Shell 沙箱:Low 完整性级别 + +`auto` 确认模式下,`run_shell` 在 Low 完整性级别的受限令牌里运行(界面显示 `auto · lowil`): + +- 子进程的令牌去掉了管理员组和特权,完整性级别为 Low(S-1-16-4096),放进 Job Object, + 超时或按 Esc 时整个进程树一起结束。 +- Low 进程只能写标为 Low 的位置。第一次在某个工作区跑沙箱命令时,CST-Agent 给工作区 + 加上可继承的 Low 标签(`S:(ML;OICI;NW;;;LW)`),并在输出里提示一次; + 记录写进 `sandbox-labels`。沙箱的临时目录是 `%TEMP%\cst-agent-sandbox`(同样标为 Low), + 子进程的 `TEMP` / `TMP` 指向这里。 +- 不会给盘符根目录、用户目录本身、`%APPDATA%`、`%LOCALAPPDATA%`、`%TEMP%` 本身, + 以及 Windows、Program Files、ProgramData 下面的目录打标签:在这些地方启动时,命令不会运行, + 界面会问是否在沙箱外重跑。请在项目目录里启动。 +- `/sandbox` 列出打过标签的目录;`/sandbox reset-labels` 恢复它们原来的标签。 +- 沙箱里可以读、执行用户能访问的文件(凭据文件除外),**不限制联网**。所以像联网的命令 + (`curl`、`wget`、`ssh`、`git clone/fetch/pull/push`、`npm install`、`pip install`、 + `winget install`、`Invoke-WebRequest` 等)在 `auto` 模式下仍然会弹窗确认。 +- 命令像是被拦了(`Access is denied`、`拒绝访问`、`Permission denied` 等)时,会问 + “Rerun outside the sandbox?”:`y` 在沙箱外重跑一次,`p` 保存命令前缀规则并重跑,`n` 保留结果。 +- MSYS2 / Cygwin 的程序(包括 Git for Windows 自带的 `usr\bin` 工具)在 Low 级别下无法初始化 + (`0xc0000142`),同样会触发上面的重跑询问。Git for Windows 的 `git.exe` 本身不受影响。 +- `CST_SANDBOX=off` 关闭沙箱(`auto` 退回只读白名单规则);`CST_SANDBOX=lowil` 只用 Low IL。 + +它防的是误操作写坏工作区以外的文件,不是对抗恶意程序的安全边界。 + +## 从源码构建 + +两种方式产物一样:`dist/windows-x86_64/` 和 `dist/cst-agent-VERSION-windows-x86_64.zip`。 +第三方依赖(curl、ncurses、busybox-w32)都按固定版本从源码编译、静态链接。 + +### 在 Linux 上交叉编译 + +```bash +git clone https://git.whcself.xyz/WHC/CST-agent.git +cd CST-agent +scripts/bootstrap_windows.sh +``` + +NixOS / 有 Nix 时自动进 `shell-win.nix`(`pkgsCross.ucrt64`);否则需要 UCRT 版的 +`x86_64-w64-mingw32-gcc`(带 winpthreads)、cmake、ninja、git、perl。 + +### 在 Windows 上用 MSYS2 原生编译 + +装好 [MSYS2](https://www.msys2.org/) 后,打开 **MSYS2 UCRT64** 终端(不是 MINGW64: +MINGW64 链接 msvcrt,UTF-8 清单不起作用): + +```bash +pacman -S --needed git +git clone https://git.whcself.xyz/WHC/CST-agent.git +cd CST-agent +scripts/bootstrap_msys2.sh +``` + +脚本会用 pacman 装 UCRT64 的 gcc / cmake / ninja,编译依赖和 `cst-agent.exe`,跑 ctest +(`CST_SKIP_TESTS=1` 跳过),再打包。 + +### 测试 + +`scripts/win_test.sh` 在本机 QEMU 里的 Windows Server 虚拟机(英文 / 中文系统,管理员 / +普通用户 / 中文用户名)上交叉编译后跑全部测试;虚拟机的创建和控制见 `scripts/winvm/`。 + +### 发布 + +```bash +GITEA_TOKEN=... scripts/release_windows.sh --build +``` + +创建(或复用)`vVERSION` Release,上传 zip 和 `.sha256`。token 只从环境变量读取,经 stdin +交给 curl,不会出现在命令行参数、磁盘或输出里。 + +## 已知限制 + +- 英文等非中文系统的传统控制台默认用 Consolas,它没有中文字形,中文会显示成方框(列宽仍然正确)。 + 用 Windows Terminal,或在控制台“属性 → 字体”里选新宋体等中文字体即可。 +- 传统控制台里输入表情符号(BMP 以外的字符)可能显示不对;中文输入没问题。 +- 通过 SSH / ConPTY 远程运行时,鼠标事件不会转发过来,键盘操作不受影响。 +- 很大的工作区第一次打标签要等一会儿(标签要继承到每个文件)。 +- 只有 x86_64 发行包。 diff --git a/docs/compat.md b/docs/compat.md index 57baf24..052cf4a 100644 --- a/docs/compat.md +++ b/docs/compat.md @@ -48,12 +48,13 @@ write beneath the workspace, the temp directory, `/dev/null` and `/dev/tty`. | Linux 5.13+ | Landlock (raw syscalls, no extra library) | ABI ≥ 4 (6.7+) also blocks TCP connect/bind | | Linux 5.5+ without Landlock (Android / Termux, WSL, older distros), x86_64 / aarch64 | seccomp user-notify | refuses IPv4/IPv6 sockets; see below | | macOS | `sandbox_init` profile | also denies network | -| Older kernels, Windows | none | falls back to the read-only rules | +| Windows 10 1903+ | Low integrity restricted token + Job Object | writes need a Low label (given to the workspace once); network is not blocked, so network commands still ask; see `README-windows.md` | +| Older kernels | none | falls back to the read-only rules | The status bar and sidebar show `auto · landlock`, `auto · seccomp`, -`auto · seatbelt` or `auto · rules`. `CST_SANDBOX=off` disables the -sandbox; `CST_SANDBOX=landlock` or `CST_SANDBOX=seccomp` allows only that -kind. When a sandboxed command looks blocked, the TUI asks whether to rerun +`auto · seatbelt`, `auto · lowil` or `auto · rules`. `CST_SANDBOX=off` +disables the sandbox; `CST_SANDBOX=landlock`, `seccomp` or `lowil` allows +only that kind. When a sandboxed command looks blocked, the TUI asks whether to rerun it outside the sandbox. The policy is built in the parent; the forked child only makes a few async-signal-safe syscalls before `exec`, so the agent itself is never confined. @@ -140,7 +141,7 @@ TUI: - [ ] Tab toggles Build/Plan, Shift+Tab cycles ask/auto/yolo, Ctrl+P opens the palette - [ ] `/model` lists the provider's `/models` (marked `(api)`); a wrong key is re-asked - [ ] `auto` mode: `echo x > ~/outside` in run_shell asks "Rerun outside the - sandbox?" (Linux/Termux/macOS); with `CST_SANDBOX=off`, `ls` runs and `touch x` asks + sandbox?" (Linux/Termux/macOS/Windows); with `CST_SANDBOX=off`, `ls` runs and `touch x` asks - [ ] Resize the terminal during streaming: layout recovers - [ ] `NO_COLOR=1` and `CST_ASCII=1` render readable screens @@ -149,6 +150,23 @@ WSL specifics: - [ ] Workspace under `/mnt/c/...` works (slower IO is expected); `~/project` is recommended - [ ] Windows Terminal: mouse wheel scroll and OSC 52 `/copy` +Windows specifics (`scripts/win_test.sh` runs the tests on English and +Chinese VMs as Administrator, a standard user and a Chinese user name): + +- [ ] `install_windows.ps1 -Zip dist\cst-agent-*.zip` installs, adds the user + PATH; `cst-agent-update -Zip ...` works while `cst-agent.exe` runs +- [ ] conhost with a raster font: Chinese renders (font switched), and the + old font is back after exit; Windows Terminal renders the same +- [ ] Chinese input via IME, paste, mouse click / drag-copy / wheel +- [ ] Workspace under `C:\Users\<中文名>\项目`: read/write/edit/apply_patch + keep CRLF and BOM; `run_shell` output in Chinese is readable on a + 936 system +- [ ] `auto · lowil`: first command reports the label; `echo x > C:\outside` + asks to rerun; `curl ...` asks before running; `/sandbox reset-labels` + restores the label +- [ ] Esc during `sleep 30 & sleep 30` returns at once and leaves no + `busybox.exe` behind + macOS specifics: - [ ] Builds with Homebrew ncurses and with the system ncurses (`brew unlink ncurses`) diff --git a/docs/windows_selftest.md b/docs/windows_selftest.md index d84a998..abaf997 100644 --- a/docs/windows_selftest.md +++ b/docs/windows_selftest.md @@ -1,11 +1,9 @@ # Windows self-test checklist -Run on a Windows machine (or wine) after MinGW build + busybox.exe beside the binary. +Automated: `scripts/win_test.sh` cross-builds and runs every test on the +English and Chinese Windows Server VMs as Administrator, a standard user and +a Chinese user name (VM tooling in `scripts/winvm/`). -1. `cst-agent.exe --help` exits 0 -2. `set CST_API_KEY=...` then `--headless --workspace . --prompt "run echo hello"` — tool output contains hello -3. `write_file` under workspace succeeds; path outside workspace rejected -4. Credentials saved under `%APPDATA%\cst-agent\credentials` -5. With PDCurses build: launch without `--headless`, confirm TUI draws, `/help` works, Esc cancels - -Shell must resolve to bundled busybox unless `CST_SHELL` is set. +Manual: the "Windows specifics" list in [compat.md](compat.md), plus the +general CLI and TUI items there. Install and sandbox details are in +[README-windows.md](README-windows.md). diff --git a/include/cst_perms.h b/include/cst_perms.h index 09c2c3e..8f86fea 100644 --- a/include/cst_perms.h +++ b/include/cst_perms.h @@ -26,6 +26,9 @@ void cst_perms_suggest_prefix(const char *cmd, char *out, size_t out_sz); /* 1 when every segment of cmd (split on | || && ;) is a read-only command * from a fixed whitelist, with no output redirection or substitution. */ int cst_shell_is_safe(const char *cmd); +/* 1 when cmd looks like it talks to the network (curl, ssh, git push, + * npm install, ...). Errs on the side of yes. */ +int cst_shell_uses_network(const char *cmd); #ifdef __cplusplus } diff --git a/include/cst_sandbox.h b/include/cst_sandbox.h index 555d63f..c104b05 100644 --- a/include/cst_sandbox.h +++ b/include/cst_sandbox.h @@ -12,6 +12,10 @@ * against mistakes, not hostile code (the path is checked before * the kernel reads it again). * macOS sandbox_init (Seatbelt profile) + * Windows Low integrity level: a restricted token (no admin privileges, + * Low IL) can only write objects labeled Low, so the workspace and + * a temp dir get that label; the credentials file is labeled + * no-read-up. The network stays open (callers ask first). * other none: callers fall back to rule-based approval */ @@ -27,15 +31,16 @@ typedef enum { CST_SANDBOX_NONE = 0, CST_SANDBOX_LANDLOCK, CST_SANDBOX_SEATBELT, - CST_SANDBOX_SECCOMP + CST_SANDBOX_SECCOMP, + CST_SANDBOX_LOWIL } cst_sandbox_kind_t; /* What this process can use; cached. CST_SANDBOX=off|0|none forces NONE, - * CST_SANDBOX=landlock|seccomp allows only that kind. */ + * CST_SANDBOX=landlock|seccomp|lowil allows only that kind. */ int cst_sandbox_probe(void); /* Drops the cache and probes again (tests that change CST_SANDBOX). */ int cst_sandbox_reprobe(void); -/* "landlock" | "seatbelt" | "seccomp" | "none" */ +/* "landlock" | "seatbelt" | "seccomp" | "lowil" | "none" */ const char *cst_sandbox_name(int kind); /* Landlock ABI version (0 when unavailable); ≥4 also blocks TCP. */ int cst_sandbox_landlock_abi(void); @@ -45,6 +50,9 @@ typedef struct { int ruleset_fd; /* Landlock, built in the parent */ char *profile; /* Seatbelt */ void *sc; /* seccomp: filter program and supervisor thread */ + void *token; /* lowil: primary token for CreateProcessAsUserW */ + char *tmpdir; /* lowil: Low-labeled TEMP for the child */ + char *note; /* lowil: shown once when a directory was just labeled */ } cst_sandbox_plan_t; /* Parent side: build the policy. Returns 0, or -1 (plan->kind = NONE). */ @@ -64,6 +72,14 @@ int cst_sandbox_run_shell(const char *workspace, const char *cmd, * way (EACCES / EPERM / EROFS / blocked network). */ int cst_sandbox_blocked(const char *out, size_t n, int exit_code); +/* lowil: directories this user's cst-agent labeled Low (recorded in the + * config dir), one per line; malloc'd, NULL when there are none. */ +char *cst_sandbox_labels(void); +/* Gives every recorded directory its old label back. Returns how many were + * restored (-1 when some failed; those stay recorded); *report gets a + * malloc'd summary. */ +int cst_sandbox_reset_labels(char **report); + #ifdef __cplusplus } #endif diff --git a/include/cst_util.h b/include/cst_util.h index 751ed39..abf922e 100644 --- a/include/cst_util.h +++ b/include/cst_util.h @@ -25,6 +25,27 @@ char *cst_buf_steal(cst_buf_t *b); size_t cst_utf8_floor(const char *s, size_t n); /* Copy s[0..n) replacing invalid UTF-8 with '?'. malloc'd, NUL-terminated. */ char *cst_utf8_sanitize(const char *s, size_t n, size_t *out_len); +int cst_utf8_valid(const char *s, size_t n); + +/* Text files may start with a UTF-8 BOM (Windows editors): its length (0/3), + * and an in-place strip of the NUL-terminated s (*len updated when given). */ +size_t cst_bom_len(const char *s, size_t n); +void cst_strip_bom(char *s, size_t *len); +/* Line endings: "\r\n" -> "\n" in place (returns the new length); whether + * every line of s ends in "\r\n" (and at least one does); and a malloc'd + * copy of s with bare "\n" turned into "\r\n" when crlf, its BOM dropped and + * a BOM added when bom. Edits keep a Windows file's style with these. */ +size_t cst_crlf_to_lf(char *s, size_t n); +int cst_text_is_crlf(const char *s, size_t n); +char *cst_text_restyle(const char *s, size_t n, int crlf, int bom, + size_t *out_len); + +/* fnmatch(3) without FNM_PATHNAME ('*' also crosses '/'): * ? [..] [!..] + * [[:class:]] and backslash escapes over UTF-8 code points. Returns 0 on a + * match. Always case-insensitive on Windows. */ +#define CST_FNM_PERIOD 1 /* a leading '.' must be matched literally */ +#define CST_FNM_CASEFOLD 2 +int cst_fnmatch(const char *pat, const char *str, int flags); uint64_t cst_fnv1a64(uint64_t h, const void *data, size_t n); #define CST_FNV_INIT 1469598103934665603ULL diff --git a/include/cst_version.h b/include/cst_version.h index 75d6277..ec6ac5e 100644 --- a/include/cst_version.h +++ b/include/cst_version.h @@ -1,6 +1,14 @@ #ifndef CST_VERSION_H #define CST_VERSION_H -#define CST_VERSION "0.5.0" +#define CST_VERSION_MAJOR 0 +#define CST_VERSION_MINOR 6 +#define CST_VERSION_PATCH 0 + +#define CST_VERSION_STR_(x) #x +#define CST_VERSION_STR(x) CST_VERSION_STR_(x) +#define CST_VERSION \ + CST_VERSION_STR(CST_VERSION_MAJOR) \ + "." CST_VERSION_STR(CST_VERSION_MINOR) "." CST_VERSION_STR(CST_VERSION_PATCH) #endif diff --git a/include/os_compat.h b/include/os_compat.h index 4d72c7f..c70b351 100644 --- a/include/os_compat.h +++ b/include/os_compat.h @@ -22,6 +22,28 @@ int os_realpath(const char *path, char *out, size_t out_sz); /* True if path is under root (after realpath/norm). Trailing slash rules apply. */ int os_path_under(const char *root, const char *path); +/* --- directory listing / file type --- */ +typedef struct os_dir os_dir_t; +typedef struct { + const char *name; /* UTF-8; valid until the next os_dir_next */ + int is_dir; + int is_link; /* symlink, or a junction on Windows (not followed) */ +} os_dirent_t; +os_dir_t *os_dir_open(const char *path); +/* Next entry, skipping "." and ".."; NULL at the end. */ +const os_dirent_t *os_dir_next(os_dir_t *d); +void os_dir_close(os_dir_t *d); + +typedef struct { + int is_dir, is_reg, is_link; + long long size; + double mtime; /* seconds since the epoch */ +} os_stat_t; +/* Like lstat(2): a link is reported as a link, not followed. */ +int os_lstat(const char *path, os_stat_t *st); +/* Replace `to` if it exists (atomic where the OS allows). */ +int os_rename_replace(const char *from, const char *to); + /* --- dirs / config --- */ int os_mkdir(const char *path, int mode); int os_mkdir_p(const char *path, int mode); @@ -67,6 +89,28 @@ int os_run_shell_ex(const char *cwd, const char *cmd, size_t max_out, os_run_result_t *r, os_child_hook_fn child_hook, void *ud); +#ifdef _WIN32 +/* Windows: run under `token` (a primary token, e.g. the Low-IL sandbox + * token; NULL = ours) with env_add "KEY=VALUE" entries layered on the + * inherited environment. Every child runs in a kill-on-close Job Object, so + * timeouts and cancels end the whole process tree. */ +int os_run_capture_token(const char *cwd, const char *const argv[], + size_t max_out, os_run_result_t *r, void *token, + const char *const env_add[]); +/* os_run_shell_c under token, with env_add. */ +int os_run_shell_token(const char *cwd, const char *cmd, size_t max_out, + os_run_result_t *r, void *token, + const char *const env_add[]); +/* "Windows 10.0.26100 x86_64" (the real version, not the compat shim's). */ +void os_windows_version(char *out, size_t n); +/* Sets the mandatory-label SACL, e.g. "S:(ML;OICI;NW;;;LW)"; 0 / -1. */ +int os_win_set_label(const char *path, const char *sddl); +/* The explicit (not inherited) label as SDDL, "S:" when there is none. */ +int os_win_get_label(const char *path, char *out, size_t n); +/* CF_UNICODE_TEXT clipboard (the console may not do OSC 52). */ +int os_clipboard_set(const char *utf8, size_t n); +#endif + /* Child processes get stdin=/dev/null, their own process group, and are * killed after the timeout (CST_SHELL_TIMEOUT seconds, default 120) or when * *flag becomes non-zero. Exit code 124 = timeout, 130 = cancelled. */ @@ -80,7 +124,7 @@ int os_get_run_timeout(void); * the inherited environment (NULL-terminated, may be NULL). */ typedef struct { #ifdef _WIN32 - void *proc, *in_w, *out_r; + void *proc, *in_w, *out_r, *job; #else int pid, in_fd, out_fd; #endif @@ -106,10 +150,30 @@ void os_run_parallel(os_task_fn fn, void *const args[], int n); /* Monotonic milliseconds (arbitrary epoch). */ double os_now_ms(void); +/* --- text encoding --- + * cst-agent is UTF-8 throughout. On Windows the manifest makes UTF-8 the + * process code page; os_console_init() also switches the console to UTF-8 + * (restored at exit), sets a UTF-8 C locale and asks Python children for + * UTF-8. Returns -1 when the process code page is not UTF-8 (pre-1903 + * Windows); no-op returning 0 elsewhere. */ +int os_console_init(void); +/* Child output that is not valid UTF-8, decoded from the console/OEM/ANSI + * code page: malloc'd UTF-8, or NULL when s is already UTF-8 (or not on + * Windows). */ +char *os_output_to_utf8(const char *s, size_t n, size_t *out_len); +#ifdef _WIN32 +#include +/* malloc'd conversions; NULL on invalid input or allocation failure */ +wchar_t *os_utf8_to_wide(const char *s); +char *os_wide_to_utf8(const wchar_t *w); +#endif + /* --- misc --- */ void os_localtime(const time_t *t, struct tm *out); int os_getpid(void); -int os_chmod_private(const char *path); /* best-effort 0600; no-op/ok on Win */ +/* 0600; on Windows a Medium no-read-up label, so Low-IL children (the + * shell sandbox) cannot read the file. */ +int os_chmod_private(const char *path); /* --- UTF-8 -------------------------------------------------------------- * Encode one code point as UTF-8, independent of the C locale. diff --git a/res/windows/cst-agent.manifest b/res/windows/cst-agent.manifest new file mode 100644 index 0000000..8e464ca --- /dev/null +++ b/res/windows/cst-agent.manifest @@ -0,0 +1,23 @@ + + + + + + UTF-8 + true + + + + + + + + + + + + + + + diff --git a/res/windows/cst-agent.rc b/res/windows/cst-agent.rc new file mode 100644 index 0000000..78633a6 --- /dev/null +++ b/res/windows/cst-agent.rc @@ -0,0 +1,29 @@ +#include +#include "cst_version.h" + +1 24 "cst-agent.manifest" + +VS_VERSION_INFO VERSIONINFO +FILEVERSION CST_VERSION_MAJOR, CST_VERSION_MINOR, CST_VERSION_PATCH, 0 +PRODUCTVERSION CST_VERSION_MAJOR, CST_VERSION_MINOR, CST_VERSION_PATCH, 0 +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_APP +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "WHC" + VALUE "FileDescription", "CST agent" + VALUE "FileVersion", CST_VERSION + VALUE "InternalName", "cst-agent" + VALUE "OriginalFilename", "cst-agent.exe" + VALUE "ProductName", "CST agent" + VALUE "ProductVersion", CST_VERSION + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/scripts/bootstrap_msys2.sh b/scripts/bootstrap_msys2.sh new file mode 100755 index 0000000..0ad8036 --- /dev/null +++ b/scripts/bootstrap_msys2.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Native Windows build from an MSYS2 UCRT64 shell (https://www.msys2.org/). +# scripts/bootstrap_msys2.sh [repo-dir] +# Installs the UCRT64 toolchain with pacman, builds the same pinned static +# deps as the Linux cross build (curl with Schannel, ncurses with the Windows +# console patches, busybox-w32 Unicode), then a static cst-agent.exe with the +# UTF-8 manifest, runs the tests and packages dist/windows-x86_64/. +# CST_SKIP_TESTS=1 skips ctest. +set -euo pipefail + +if [[ "${MSYSTEM:-}" != UCRT64 ]]; then + echo "run this from the MSYS2 UCRT64 shell: the UTF-8 manifest needs the" \ + "UCRT runtime (MINGW64 links msvcrt)" >&2 + exit 1 +fi + +REPO_URL="${CST_AGENT_REPO:-https://git.whcself.xyz/WHC/CST-agent.git}" +DIR="${1:-}" +if [[ -z "$DIR" ]]; then + if [[ -f CMakeLists.txt && -d src ]]; then + DIR="$(pwd)" + else + DIR="CST-agent" + fi +fi + +pacman -S --needed --noconfirm \ + mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-cmake \ + mingw-w64-ucrt-x86_64-ninja make git patch tar perl zip diffutils + +if [[ ! -f "$DIR/CMakeLists.txt" ]]; then + echo "==> git clone $REPO_URL -> $DIR" + git clone "$REPO_URL" "$DIR" +fi +cd "$DIR" + +# third-party script: unprefixed native tools +export MINGW_HOST= BUILD_CC=gcc +[[ -f third_party/prefix-mingw/lib/libcurl.a ]] || scripts/build_third_party_mingw.sh curl +[[ -f third_party/prefix-mingw/lib/libncursesw.a ]] || scripts/build_third_party_mingw.sh ncurses +if [[ ! -f third_party/busybox/busybox.exe ]]; then + scripts/build_third_party_mingw.sh busybox-w32 || scripts/fetch_busybox_w32.sh +fi + +echo "==> cmake (MSYS2 UCRT64, static, TUI)" +cmake -S . -B build-msys2 -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$PWD/third_party/prefix-mingw" \ + -DCST_WIN_STATIC=ON -DCST_BUILD_TUI=ON +cmake --build build-msys2 + +if [[ -z "${CST_SKIP_TESTS:-}" ]]; then + echo "==> ctest" + cp -f third_party/busybox/busybox.exe build-msys2/ + ctest --test-dir build-msys2 --output-on-failure +fi + +echo "==> package" +CST_WIN_BUILD=build-msys2 scripts/package_windows.sh diff --git a/scripts/bootstrap_windows.sh b/scripts/bootstrap_windows.sh index 1e31209..c031116 100755 --- a/scripts/bootstrap_windows.sh +++ b/scripts/bootstrap_windows.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -# One-shot on a Linux host: cross-build Windows x86_64 release (MinGW). -# Output: dist/windows-x86_64/ — copy that folder to a Windows machine. +# Linux host: cross-build the Windows x86_64 release (mingw-w64, UCRT, static). +# scripts/bootstrap_windows.sh [repo-dir] +# With Nix the toolchain comes from shell-win.nix; otherwise x86_64-w64-mingw32-gcc +# (UCRT flavour, with winpthreads), cmake, ninja, git and perl must be installed. +# Output: dist/windows-x86_64/ and dist/cst-agent-VERSION-windows-x86_64.zip set -euo pipefail REPO_URL="${CST_AGENT_REPO:-https://git.whcself.xyz/WHC/CST-agent.git}" DIR="${1:-}" - if [[ -z "$DIR" ]]; then if [[ -f CMakeLists.txt && -d src ]]; then DIR="$(pwd)" @@ -13,36 +15,33 @@ if [[ -z "$DIR" ]]; then DIR="CST-agent" fi fi - if [[ ! -f "$DIR/CMakeLists.txt" ]]; then echo "==> git clone $REPO_URL -> $DIR" git clone "$REPO_URL" "$DIR" fi cd "$DIR" -need() { command -v "$1" >/dev/null || { echo "missing $1 (install mingw-w64 gcc cmake git)" >&2; exit 1; }; } -need git; need cmake; need x86_64-w64-mingw32-gcc +if ! command -v x86_64-w64-mingw32-gcc >/dev/null && [[ -z "${IN_NIX_SHELL:-}" ]] && + command -v nix-shell >/dev/null; then + exec nix-shell shell-win.nix --run "$(printf '%q ' "$0" "$PWD")" +fi -echo "==> third-party (curl Schannel + PDCurses + busybox-w32)" -chmod +x scripts/*.sh 2>/dev/null || true -./scripts/build_third_party_mingw.sh all +need() { command -v "$1" >/dev/null || { echo "missing $1 (mingw-w64 UCRT gcc, cmake, ninja, git, perl)" >&2; exit 1; }; } +need git; need cmake; need ninja; need perl; need x86_64-w64-mingw32-gcc +if ! printf '#include \n' | x86_64-w64-mingw32-gcc -dM -E - | grep -qw _UCRT; then + echo "x86_64-w64-mingw32-gcc targets msvcrt; the UTF-8 manifest needs a UCRT toolchain" >&2 + exit 1 +fi -echo "==> cmake Windows TUI" -cmake -B build-win-tui \ +[[ -f third_party/prefix-mingw/lib/libcurl.a ]] || scripts/build_third_party_mingw.sh curl +[[ -f third_party/prefix-mingw/lib/libncursesw.a ]] || scripts/build_third_party_mingw.sh ncurses +[[ -f third_party/busybox/busybox.exe ]] || scripts/build_third_party_mingw.sh busybox-w32 + +echo "==> cmake (Windows, TUI)" +cmake -S . -B build-win-tui -G Ninja \ -DCMAKE_TOOLCHAIN_FILE=cmake/mingw-w64-x86_64.cmake \ - -DCST_BUILD_TUI=ON \ - -DCMAKE_PREFIX_PATH="$PWD/third_party/prefix-mingw" \ - -DPDCurses_ROOT="$PWD/third_party/prefix-mingw" -cmake --build build-win-tui -j"$(nproc 2>/dev/null || echo 4)" + -DCMAKE_BUILD_TYPE=Release -DCST_BUILD_TUI=ON +cmake --build build-win-tui echo "==> package" -./scripts/package_windows.sh - -echo -echo "OK: dist/windows-x86_64/" -echo "Copy that folder to Windows, then:" -echo " set OPENAI_API_KEY=sk-..." -echo " set OPENAI_BASE_URL=https://api.openai.com/v1" -echo " set OPENAI_MODEL=gpt-4.1-mini" -echo " cst-agent.exe --workspace ." -echo "See docs/env_openai.md and docs/README-windows.md" +scripts/package_windows.sh diff --git a/scripts/build_third_party_mingw.sh b/scripts/build_third_party_mingw.sh index f51e05b..5874d3b 100755 --- a/scripts/build_third_party_mingw.sh +++ b/scripts/build_third_party_mingw.sh @@ -1,139 +1,127 @@ #!/usr/bin/env bash -# Build third-party deps for Windows MinGW cross + optional Unix busybox. -# Sources: -# busybox-w32: https://github.com/rmyorston/busybox-w32.git -# busybox: https://github.com/mirror/busybox.git -# curl: https://github.com/curl/curl.git (Windows via Schannel) -# PDCurses: https://github.com/wmcbrine/PDCurses.git +# Build pinned third-party deps for Windows x86_64 (mingw-w64, UCRT) into +# third_party/prefix-mingw, plus busybox-w32 into third_party/busybox. +# nix-shell shell-win.nix --run 'scripts/build_third_party_mingw.sh [all|curl|pdcurses|ncurses|busybox-w32]' +# Works the same from an MSYS2 UCRT64 shell (MINGW_HOST= uses unprefixed tools). set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" TP="$ROOT/third_party" -HOST="${MINGW_HOST:-x86_64-w64-mingw32}" +PREFIX="$TP/prefix-mingw" +HOST="${MINGW_HOST-x86_64-w64-mingw32}" +T="${HOST:+$HOST-}" JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}" -mkdir -p "$TP" +CURL_TAG=curl-8_22_0 +PDCURSES_TAG=3.9 +NCURSES_VER=6.6 +BUSYBOX_W32_REV=9b50a1e + +mkdir -p "$TP" "$PREFIX" need_cmd() { command -v "$1" >/dev/null || { echo "missing: $1" >&2; exit 1; }; } need_cmd git need_cmd make -need_cmd "${HOST}-gcc" +need_cmd "${T}gcc" -clone_or_update() { - local url="$1" dir="$2" - if [[ -d "$dir/.git" ]]; then - git -C "$dir" fetch --depth 1 origin 2>/dev/null || true - git -C "$dir" pull --ff-only 2>/dev/null || true - else - git clone --depth 1 "$url" "$dir" +checkout() { # url dir rev + local url="$1" dir="$2" rev="$3" + [[ -d "$dir/.git" ]] || git clone -q --filter=blob:none "$url" "$dir" + if ! git -C "$dir" rev-parse -q --verify "$rev^{commit}" >/dev/null; then + git -C "$dir" fetch -q --tags origin fi + git -C "$dir" checkout -q --detach "$rev" } -# --- curl (static, Schannel) --- -build_curl_mingw() { - clone_or_update https://github.com/curl/curl.git "$TP/curl" - cd "$TP/curl" - if [[ ! -f configure ]]; then - autoreconf -fi 2>/dev/null || true - fi - # Prefer cmake for curl modern tree - rm -rf build-mingw - cmake -B build-mingw -G "Unix Makefiles" \ - -DCMAKE_SYSTEM_NAME=Windows \ - -DCMAKE_C_COMPILER="${HOST}-gcc" \ - -DCMAKE_RC_COMPILER="${HOST}-windres" \ - -DCMAKE_INSTALL_PREFIX="$TP/prefix-mingw" \ - -DBUILD_SHARED_LIBS=OFF \ - -DCURL_USE_SCHANNEL=ON \ - -DCURL_USE_OPENSSL=OFF \ - -DCURL_DISABLE_LDAP=ON \ - -DCURL_USE_LIBPSL=OFF \ - -DCURL_ZLIB=OFF \ - -DCURL_BROTLI=OFF \ - -DCURL_ZSTD=OFF \ - -DUSE_NGHTTP2=OFF \ - -DBUILD_CURL_EXE=OFF \ - -DBUILD_TESTING=OFF \ - -DHTTP_ONLY=OFF - cmake --build build-mingw -j"$JOBS" - cmake --install build-mingw - echo "curl installed -> $TP/prefix-mingw" +build_curl() { + checkout https://github.com/curl/curl.git "$TP/curl" "$CURL_TAG" + rm -rf "$TP/curl/build-mingw" + local tc=() + [[ -n "$HOST" ]] && tc=(-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_C_COMPILER="${T}gcc" -DCMAKE_RC_COMPILER="${T}windres") + cmake -S "$TP/curl" -B "$TP/curl/build-mingw" -G Ninja "${tc[@]}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ + -DCURL_USE_SCHANNEL=ON -DCURL_USE_OPENSSL=OFF \ + -DCURL_WINDOWS_SSPI=ON -DENABLE_UNICODE=ON \ + -DCURL_DISABLE_LDAP=ON -DCURL_USE_LIBPSL=OFF -DCURL_USE_LIBSSH2=OFF \ + -DUSE_LIBIDN2=OFF -DUSE_NGHTTP2=OFF \ + -DCURL_ZLIB=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF \ + -DBUILD_CURL_EXE=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF \ + -DBUILD_LIBCURL_DOCS=OFF -DBUILD_MISC_DOCS=OFF -DENABLE_CURL_MANUAL=OFF + cmake --build "$TP/curl/build-mingw" -j"$JOBS" + cmake --install "$TP/curl/build-mingw" + echo "curl $CURL_TAG -> $PREFIX" } -# --- PDCurses --- -build_pdcurses_mingw() { - clone_or_update https://github.com/wmcbrine/PDCurses.git "$TP/PDCurses" +build_pdcurses() { + checkout https://github.com/wmcbrine/PDCurses.git "$TP/PDCurses" "$PDCURSES_TAG" cd "$TP/PDCurses/wincon" - make -f Makefile clean 2>/dev/null || true - make -f Makefile \ - CC="${HOST}-gcc" \ - LINK="${HOST}-gcc" \ - CFLAGS="-O2 -I.. -DPDC_WIDE -DPDC_FORCE_UTF8" \ - LIBEXE="${HOST}-ar" \ - LIBFLAGS=rcv \ - pdcurses.a - mkdir -p "$TP/prefix-mingw/include" "$TP/prefix-mingw/lib" - cp -f ../curses.h "$TP/prefix-mingw/include/" - [[ -f ../panel.h ]] && cp -f ../panel.h "$TP/prefix-mingw/include/" || true - cp -f pdcurses.a "$TP/prefix-mingw/lib/libpdcurses.a" - # Also install as curses for find_library - cp -f pdcurses.a "$TP/prefix-mingw/lib/libcurses.a" - echo "PDCurses -> $TP/prefix-mingw" + make clean >/dev/null 2>&1 || true + make -j"$JOBS" CC="${T}gcc" AR="${T}ar" STRIP="${T}strip" WIDE=Y UTF8=Y pdcurses.a + mkdir -p "$PREFIX/include/pdcurses" "$PREFIX/lib" + cp -f ../curses.h ../panel.h "$PREFIX/include/pdcurses/" + cp -f pdcurses.a "$PREFIX/lib/libpdcurses.a" + echo "PDCurses $PDCURSES_TAG -> $PREFIX" +} + +build_ncurses() { + local src="$TP/ncurses-$NCURSES_VER" tgz="$TP/ncurses-$NCURSES_VER.tar.gz" + [[ -f "$tgz" ]] || curl -fL -o "$tgz" \ + "https://invisible-island.net/archives/ncurses/ncurses-$NCURSES_VER.tar.gz" + [[ -d "$src" ]] || tar -C "$TP" -xzf "$tgz" + local pf + for pf in "$ROOT"/scripts/patches/ncurses-"$NCURSES_VER"-*.patch; do + [[ -e "$pf" ]] || continue + patch -d "$src" -p1 -R --dry-run -s -f <"$pf" >/dev/null 2>&1 && continue + if ! patch -d "$src" -p1 --dry-run -s -f <"$pf" >/dev/null 2>&1; then + echo "ncurses: $src has another version of $(basename "$pf"); re-extracting" + rm -rf "$src" + tar -C "$TP" -xzf "$tgz" + fi + patch -d "$src" -p1 -s <"$pf" + done + rm -rf "$src/build-mingw" + mkdir -p "$src/build-mingw" + cd "$src/build-mingw" + ../configure --host="${HOST:-x86_64-w64-mingw32}" --prefix="$PREFIX" \ + --without-ada --without-cxx --without-cxx-binding --without-manpages \ + --without-progs --without-tests --without-debug --without-shared \ + --enable-widec --enable-sp-funcs --enable-term-driver \ + --enable-interop --disable-home-terminfo --disable-db-install \ + --with-fallbacks=ms-terminal,xterm-256color --without-pkg-config \ + --disable-pc-files --with-build-cc="${BUILD_CC:-cc}" >/dev/null + make -j"$JOBS" libs >/dev/null + make install.libs >/dev/null + echo "ncurses $NCURSES_VER -> $PREFIX" } -# --- busybox-w32 --- build_busybox_w32() { - clone_or_update https://github.com/rmyorston/busybox-w32.git "$TP/busybox-w32" + checkout https://github.com/rmyorston/busybox-w32.git "$TP/busybox-w32" "$BUSYBOX_W32_REV" cd "$TP/busybox-w32" - # configs: try mingw64 defconfig - if [[ -f configs/mingw64_defconfig ]]; then - make mingw64_defconfig - elif [[ -f configs/mingw_defconfig ]]; then - make mingw_defconfig - else - make defconfig - fi - # Cross compile - make -j"$JOBS" CROSS_COMPILE="${HOST}-" \ - CONFIG_EXTRA_CFLAGS="-O2" \ - busybox.exe 2>&1 || make -j"$JOBS" CROSS_COMPILE="${HOST}-" + make -s mingw64u_defconfig >/dev/null + make -s -j"$JOBS" CROSS_COMPILE="$T" "$TP/busybox/LICENSE.busybox" << 'EOF' -BusyBox is GPLv2. Source: https://github.com/rmyorston/busybox-w32 -Upstream BusyBox: https://github.com/mirror/busybox + cp -f busybox.exe "$TP/busybox/busybox.exe" + cat >"$TP/busybox/LICENSE.busybox" < $TP/busybox/busybox" -} - -TARGETS="${1:-all}" -case "$TARGETS" in - curl) build_curl_mingw ;; - pdcurses) build_pdcurses_mingw ;; - busybox-w32) build_busybox_w32 ;; - busybox-unix) build_busybox_unix ;; - all) - build_curl_mingw - build_pdcurses_mingw - build_busybox_w32 - ;; - *) - echo "usage: $0 [all|curl|pdcurses|busybox-w32|busybox-unix]" - exit 1 - ;; +case "${1:-all}" in +curl) build_curl ;; +pdcurses) build_pdcurses ;; +ncurses) build_ncurses ;; +busybox-w32) build_busybox_w32 ;; +all) + build_curl + build_ncurses + build_busybox_w32 + ;; +*) + echo "usage: $0 [all|curl|pdcurses|ncurses|busybox-w32]" + exit 1 + ;; esac echo "DONE" diff --git a/scripts/install_windows.ps1 b/scripts/install_windows.ps1 new file mode 100644 index 0000000..5d4a888 --- /dev/null +++ b/scripts/install_windows.ps1 @@ -0,0 +1,106 @@ +# Install or update CST-Agent on Windows from a Gitea release. +# +# irm https://git.whcself.xyz/WHC/CST-agent/raw/branch/main/scripts/install_windows.ps1 | iex +# powershell -ExecutionPolicy Bypass -File install_windows.ps1 [-Version 0.6.0] [-Prefix DIR] [-Zip FILE] [-NoPath] +# +# Installs into %LOCALAPPDATA%\Programs\cst-agent (no admin rights needed), +# adds it to the user PATH and leaves cst-agent-update.cmd beside the exe, +# which reruns this script for that folder. Environment overrides: +# CST_VERSION, CST_PREFIX, CST_GITEA (server URL), CST_REPO (owner/name). +# Needs Windows 10 1903+ / Server 2022+ (UTF-8 process code page). +# Keep this file ASCII: Windows PowerShell 5.1 reads BOM-less scripts in the +# ANSI code page. +param( + [string]$Version = $env:CST_VERSION, + [string]$Prefix = $env:CST_PREFIX, + [string]$Zip = "", + [switch]$NoPath +) +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +$Server = if ($env:CST_GITEA) { $env:CST_GITEA.TrimEnd("/") } else { "https://git.whcself.xyz" } +$Repo = if ($env:CST_REPO) { $env:CST_REPO } else { "WHC/CST-agent" } +if (-not $Prefix) { $Prefix = Join-Path $env:LOCALAPPDATA "Programs\cst-agent" } +$Prefix = [IO.Path]::GetFullPath($Prefix) + +$build = [Environment]::OSVersion.Version.Build +if ($build -lt 18362) { + throw "CST-Agent needs Windows 10 1903 (build 18362) or newer; this is build $build." +} + +$work = Join-Path ([IO.Path]::GetTempPath()) ("cst-agent-install-" + [Guid]::NewGuid().ToString("N")) +New-Item -ItemType Directory -Force $work | Out-Null +try { + if (-not $Zip) { + $api = "$Server/api/v1/repos/$Repo/releases" + $url = if (-not $Version -or $Version -eq "latest") { "$api/latest" } else { "$api/tags/v$($Version.TrimStart('v'))" } + Write-Host "==> $url" + try { + $rel = Invoke-RestMethod -Uri $url -UseBasicParsing + } catch { + throw "no release found at $url ($($_.Exception.Message))" + } + $asset = $rel.assets | Where-Object { $_.name -like "*-windows-x86_64.zip" } | Select-Object -First 1 + if (-not $asset) { throw "release $($rel.tag_name) has no *-windows-x86_64.zip" } + $Zip = Join-Path $work $asset.name + Write-Host "==> download $($asset.name) ($($rel.tag_name))" + Invoke-WebRequest -Uri $asset.browser_download_url -OutFile $Zip -UseBasicParsing + $sum = $rel.assets | Where-Object { $_.name -eq "$($asset.name).sha256" } | Select-Object -First 1 + if ($sum) { + $sumFile = Join-Path $work $sum.name + Invoke-WebRequest -Uri $sum.browser_download_url -OutFile $sumFile -UseBasicParsing + $want = ((Get-Content $sumFile -Raw).Trim() -split "\s+")[0].ToLower() + $got = (Get-FileHash -Algorithm SHA256 $Zip).Hash.ToLower() + if ($want -ne $got) { throw "sha256 mismatch for $($asset.name): expected $want, got $got" } + Write-Host "==> sha256 ok" + } + } + + Expand-Archive -Path $Zip -DestinationPath $work -Force + $src = Join-Path $work "windows-x86_64" + if (-not (Test-Path (Join-Path $src "cst-agent.exe"))) { throw "$Zip has no windows-x86_64\cst-agent.exe" } + + New-Item -ItemType Directory -Force $Prefix | Out-Null + # A running exe cannot be overwritten but can be renamed out of the way. + foreach ($name in @("cst-agent.exe", "busybox.exe")) { + $dst = Join-Path $Prefix $name + $old = "$dst.old" + if (Test-Path $old) { Remove-Item -Force $old -ErrorAction SilentlyContinue } + if (Test-Path $dst) { + try { Remove-Item -Force $dst } catch { Rename-Item -Force $dst "$name.old" } + } + } + Copy-Item -Force -Recurse (Join-Path $src "*") $Prefix + + $update = Join-Path $Prefix "cst-agent-update.cmd" + if (-not (Test-Path $update)) { + Set-Content -Encoding Ascii -Path $update -Value @( + "@echo off", + "rem Update CST-Agent in this folder from the latest release.", + "powershell -NoProfile -ExecutionPolicy Bypass -File `"%~dp0install_windows.ps1`" -Prefix `"%~dp0.`" %*" + ) + } + if (-not (Test-Path (Join-Path $Prefix "install_windows.ps1")) -and $PSCommandPath) { + Copy-Item -Force $PSCommandPath (Join-Path $Prefix "install_windows.ps1") + } + + if (-not $NoPath) { + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") + $parts = @($userPath -split ";" | Where-Object { $_ }) + if (-not ($parts | Where-Object { $_.TrimEnd("\") -ieq $Prefix.TrimEnd("\") })) { + [Environment]::SetEnvironmentVariable("Path", (($parts + $Prefix) -join ";"), "User") + Write-Host "==> added $Prefix to the user PATH (open a new terminal to use it)" + } + if (-not (($env:Path -split ";") | Where-Object { $_.TrimEnd("\") -ieq $Prefix.TrimEnd("\") })) { + $env:Path = "$env:Path;$Prefix" + } + } + + $ver = (& (Join-Path $Prefix "cst-agent.exe") --version) 2>&1 | Select-Object -First 1 + Write-Host "==> installed $ver in $Prefix" + Write-Host "Run 'cst-agent' inside a project folder; update later with 'cst-agent-update'." +} finally { + Remove-Item -Recurse -Force $work -ErrorAction SilentlyContinue +} diff --git a/scripts/package_windows.sh b/scripts/package_windows.sh index 2ad57e1..ee7a39b 100755 --- a/scripts/package_windows.sh +++ b/scripts/package_windows.sh @@ -1,18 +1,38 @@ #!/usr/bin/env bash -# Assemble dist/windows-x86_64 from MinGW TUI build + busybox. +# Assemble dist/windows-x86_64/ and dist/cst-agent-VERSION-windows-x86_64.zip +# from the cross build (build-win-tui, else build-win; CST_WIN_BUILD=DIR +# picks another, e.g. build-msys2) and busybox-w32. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" REL="$ROOT/dist/windows-x86_64" -EXE="$ROOT/build-win-tui/cst-agent.exe" -[[ -f "$EXE" ]] || EXE="$ROOT/build-win/cst-agent.exe" +if [[ -n "${CST_WIN_BUILD:-}" ]]; then + EXE="$ROOT/$CST_WIN_BUILD/cst-agent.exe" +else + EXE="$ROOT/build-win-tui/cst-agent.exe" + [[ -f "$EXE" ]] || EXE="$ROOT/build-win/cst-agent.exe" +fi BB="$ROOT/third_party/busybox/busybox.exe" -DLL="/usr/x86_64-w64-mingw32/bin/libwinpthread-1.dll" +[[ -f "$EXE" ]] || { echo "no cst-agent.exe; build first" >&2; exit 1; } +[[ -f "$BB" ]] || { echo "no busybox.exe; scripts/build_third_party_mingw.sh busybox-w32" >&2; exit 1; } +ver="$(sed -n 's/^#define CST_VERSION_\(MAJOR\|MINOR\|PATCH\) \([0-9]*\)$/\2/p' "$ROOT/include/cst_version.h" | paste -sd.)" +strip="${MINGW_HOST-x86_64-w64-mingw32}" +strip="${strip:+$strip-}strip" +command -v "$strip" >/dev/null || strip=strip # MSYS2: unprefixed + +rm -rf "$REL" mkdir -p "$REL" cp -f "$EXE" "$REL/cst-agent.exe" +"$strip" "$REL/cst-agent.exe" cp -f "$BB" "$REL/busybox.exe" cp -f "$ROOT/third_party/busybox/LICENSE.busybox" "$REL/" -cp -f "$ROOT/docs/README-windows.md" "$REL/" -[[ -f "$DLL" ]] && cp -f "$DLL" "$REL/" +cp -f "$ROOT/docs/README-windows.md" "$REL/README.md" +cp -f "$ROOT/scripts/windows/cst-agent-update.cmd" "$ROOT/scripts/install_windows.ps1" "$REL/" +echo "$ver" >"$REL/VERSION" + +zip="$ROOT/dist/cst-agent-$ver-windows-x86_64.zip" +rm -f "$zip" +(cd "$ROOT/dist" && zip -qr "$zip" windows-x86_64) +(cd "$ROOT/dist" && sha256sum "$(basename "$zip")" >"$(basename "$zip").sha256") ls -la "$REL" -file "$REL/cst-agent.exe" +echo "$zip" diff --git a/scripts/patches/ncurses-6.6-win32-console.patch b/scripts/patches/ncurses-6.6-win32-console.patch new file mode 100644 index 0000000..4afe941 --- /dev/null +++ b/scripts/patches/ncurses-6.6-win32-console.patch @@ -0,0 +1,350 @@ +--- a/ncurses/curses.priv.h ++++ b/ncurses/curses.priv.h +@@ -2294,7 +2294,10 @@ + #if defined(_NC_WINDOWS_NATIVE) && !defined(_NC_MSC) && !USE_NAMED_PIPES + /* + * MinGW has wide-character functions, but they do not work correctly. ++ * Declare the originals first so a later is not macro-expanded. + */ ++#include ++#include + extern int __MINGW_NOTHROW _nc_wctomb(char *, wchar_t); + #define wctomb(s,wc) _nc_wctomb(s,wc) + #define wcrtomb(s,wc,n) _nc_wctomb(s,wc) +--- a/ncurses/tinfo/lib_win32con.c ++++ b/ncurses/tinfo/lib_win32con.c +@@ -653,6 +653,21 @@ + return result; + } + ++/* Whether handle_mouse() turns this record into an event: wheel, a button ++ * change (press or release), or a drag. */ ++static bool ++mouse_wanted(const SCREEN *sp, const MOUSE_EVENT_RECORD *mer) ++{ ++ int held = (int) (mer->dwButtonState & BUTTON_MASK); ++ ++ if (mer->dwEventFlags & MOUSE_WHEELED) ++ return TRUE; ++ if (held != sp->_drv_mouse_new_buttons) ++ return TRUE; ++ return (mer->dwEventFlags & MOUSE_MOVED) && held ++ && (sp->_mouse_mask & REPORT_MOUSE_POSITION); ++} ++ + #define AdjustY() (WINCONSOLE.buffered \ + ? 0 \ + : (int) WINCONSOLE.SBI.srWindow.Top) +@@ -661,36 +676,47 @@ + handle_mouse(SCREEN *sp, MOUSE_EVENT_RECORD mer) + { + MEVENT work; +- bool result = FALSE; ++ int old_buttons; ++ int new_buttons; + + assert(sp); + +- sp->_drv_mouse_old_buttons = sp->_drv_mouse_new_buttons; +- sp->_drv_mouse_new_buttons = mer.dwButtonState & BUTTON_MASK; ++ if (sp->_drv_mouse_tail >= FIFO_SIZE) ++ return FALSE; + +- /* +- * We're only interested if the button is pressed or released. +- * FIXME: implement continuous event-tracking. +- */ +- if (sp->_drv_mouse_new_buttons != sp->_drv_mouse_old_buttons) { +- memset(&work, 0, sizeof(work)); ++ memset(&work, 0, sizeof(work)); ++ work.x = mer.dwMousePosition.X; ++ work.y = mer.dwMousePosition.Y - AdjustY(); ++ ++ /* the wheel reports like xterm: button 4 up, button 5 down */ ++ if (mer.dwEventFlags & MOUSE_WHEELED) { ++ work.bstate = ((SHORT) HIWORD(mer.dwButtonState) > 0) ++ ? BUTTON4_PRESSED ++ : BUTTON5_PRESSED; ++ sp->_drv_mouse_fifo[sp->_drv_mouse_tail++] = work; ++ return TRUE; ++ } + +- if (sp->_drv_mouse_new_buttons) { +- work.bstate |= decode_mouse(sp, sp->_drv_mouse_new_buttons); ++ old_buttons = sp->_drv_mouse_new_buttons; ++ new_buttons = (int) (mer.dwButtonState & BUTTON_MASK); ++ sp->_drv_mouse_old_buttons = old_buttons; ++ sp->_drv_mouse_new_buttons = new_buttons; ++ ++ if (new_buttons != old_buttons) { ++ if (new_buttons & ~old_buttons) { ++ work.bstate = decode_mouse(sp, new_buttons & ~old_buttons); + } else { + /* cf: BUTTON_PRESSED, BUTTON_RELEASED */ +- work.bstate |= (decode_mouse(sp, sp->_drv_mouse_old_buttons) +- >> 1); +- result = TRUE; ++ work.bstate = decode_mouse(sp, old_buttons & ~new_buttons) >> 1; + } +- +- work.x = mer.dwMousePosition.X; +- work.y = mer.dwMousePosition.Y - AdjustY(); +- +- sp->_drv_mouse_fifo[sp->_drv_mouse_tail] = work; +- sp->_drv_mouse_tail += 1; ++ } else if ((mer.dwEventFlags & MOUSE_MOVED) && new_buttons ++ && (sp->_mouse_mask & REPORT_MOUSE_POSITION)) { ++ work.bstate = REPORT_MOUSE_POSITION; /* drag */ ++ } else { ++ return FALSE; + } +- return result; ++ sp->_drv_mouse_fifo[sp->_drv_mouse_tail++] = work; ++ return TRUE; + } + + static int +@@ -813,6 +839,13 @@ + returnCode(found); + } + ++#if USE_WIDEC_SUPPORT ++/* UTF-8 bytes of the last non-ASCII key event not yet read */ ++static int utf8_pend[4]; ++static int utf8_len, utf8_pos; ++static unsigned utf8_high; ++#endif ++ + NCURSES_EXPORT(int) + _nc_console_twait( + const SCREEN *sp, +@@ -845,6 +878,14 @@ + TR(TRACE_IEVENT, ("start twait: hdl=%p, %d milliseconds, mode: %d", + hdl, milliseconds, mode)); + ++#if USE_WIDEC_SUPPORT ++ if ((mode & TW_INPUT) && utf8_pos < utf8_len) { ++ if (timeleft) ++ *timeleft = milliseconds; ++ return TW_INPUT; ++ } ++#endif ++ + if (milliseconds < 0) + milliseconds = NC_INFINITY; + +@@ -940,9 +981,7 @@ + continue; + case MOUSE_EVENT: + T(("twait:event MOUSE_EVENT")); +- if (decode_mouse(sp, +- (inp_rec.Event.MouseEvent.dwButtonState +- & BUTTON_MASK)) == 0) { ++ if (!mouse_wanted(sp, &inp_rec.Event.MouseEvent)) { + CONSUME(); + } else if (mode & TW_MOUSE) { + code = TW_MOUSE; +@@ -1024,6 +1063,16 @@ + + T((T_CALLED("lib_win32con::_nc_console_read(%p)"), sp)); + ++#if USE_WIDEC_SUPPORT ++ /* wget_wch() decodes the input as a byte stream in the locale's ++ * multibyte encoding (UTF-8), so non-ASCII UTF-16 key events are ++ * handed over one UTF-8 byte per call. */ ++ if (utf8_pos < utf8_len) { ++ *buf = utf8_pend[utf8_pos++]; ++ returnCode(1); ++ } ++#endif ++ + while ((b = read_keycode(hdl, &inp_rec, 1, &nRead))) { + if (b && nRead > 0) { + if (rc < 0) +@@ -1034,6 +1083,36 @@ + continue; + *buf = (int) inp_rec.Event.KeyEventChar; + vk = inp_rec.Event.KeyEvent.wVirtualKeyCode; ++#if USE_WIDEC_SUPPORT ++ if (*buf >= 0x80) { ++ unsigned cp = (unsigned) *buf; ++ if (cp >= 0xD800 && cp < 0xDC00) { ++ utf8_high = cp; ++ continue; ++ } ++ if (cp >= 0xDC00 && cp < 0xE000) { ++ if (!utf8_high) ++ continue; ++ cp = 0x10000 + ((utf8_high - 0xD800) << 10) + (cp - 0xDC00); ++ } ++ utf8_high = 0; ++ utf8_len = 0; ++ if (cp < 0x800) { ++ utf8_pend[utf8_len++] = (int) (0xC0 | (cp >> 6)); ++ } else if (cp < 0x10000) { ++ utf8_pend[utf8_len++] = (int) (0xE0 | (cp >> 12)); ++ utf8_pend[utf8_len++] = (int) (0x80 | ((cp >> 6) & 0x3F)); ++ } else { ++ utf8_pend[utf8_len++] = (int) (0xF0 | (cp >> 18)); ++ utf8_pend[utf8_len++] = (int) (0x80 | ((cp >> 12) & 0x3F)); ++ utf8_pend[utf8_len++] = (int) (0x80 | ((cp >> 6) & 0x3F)); ++ } ++ utf8_pend[utf8_len++] = (int) (0x80 | (cp & 0x3F)); ++ *buf = utf8_pend[0]; ++ utf8_pos = 1; ++ break; ++ } ++#endif + /* + * There are 24 virtual function-keys, and typically + * 12 function-keys on a keyboard. Use the shift-modifier +--- a/ncurses/widechar/widechars.c ++++ b/ncurses/widechar/widechars.c +@@ -97,20 +97,10 @@ + wchar_t temp; + + if (s != NULL && n != 0) { ++ /* _nc_mbtowc() returns the bytes used, not a character count */ + count = _nc_mbtowc(&temp, s, n); +- if (count == 1) { +- int check = WideCharToMultiByte(CP_UTF8, +- 0, +- &temp, +- 1, +- NULL, +- 0, /* compute length only */ +- NULL, +- NULL); +- TR(TRACE_BITS, ("\tcheck ->%d\n", check)); +- if (check > 0 && (size_t) check <= n) { +- result = check; +- } ++ if (count > 0 && (size_t) count <= n) { ++ result = count; + } + } else { + result = 0; +--- a/ncurses/win32con/win32_driver.c ++++ b/ncurses/win32con/win32_driver.c +@@ -87,8 +87,13 @@ + if (ch & A_BOLD) + res |= FOREGROUND_INTENSITY; + +- if (ch & A_DIM) +- res |= BACKGROUND_INTENSITY; ++ /* dim lowers the foreground: bright -> normal, light gray -> dark gray */ ++ if ((ch & A_DIM) && !(ch & A_BOLD)) { ++ if (res & FOREGROUND_INTENSITY) ++ res &= ~FOREGROUND_INTENSITY; ++ else if ((res & 7) == 7) ++ res = (res & ~7) | FOREGROUND_INTENSITY; ++ } + + return res; + } +@@ -159,7 +164,6 @@ + int y, int x, cchar_t *str, int limit) + { + int actual = 0; +- MakeArray(ci, CHAR_INFO, limit); + COORD loc, siz; + SMALL_RECT rec; + int i; +@@ -169,39 +173,71 @@ + AssertTCB(); + SetSP(); + +- for (i = actual = 0; i < limit; i++) { +- ch = str[i]; +- if (isWidecExt(ch)) +- continue; +- ci[actual].CharInfoChar = CharOf(ch); +- ci[actual].Attributes = MapAttr(WINCONSOLE.SBI.wAttributes, +- AttrOf(ch)); +- if (AttrOf(ch) & A_ALTCHARSET) { +- if (_nc_wacs) { +- int which = CharOf(ch); +- if (which > 0 +- && which < ACS_LEN +- && CharOf(_nc_wacs[which]) != 0) { +- ci[actual].CharInfoChar = CharOf(_nc_wacs[which]); ++ /* ++ * The console stores a double-width character as two cells flagged ++ * leading/trailing byte. Never start or end a write inside one, and ++ * emit both halves so the columns after it stay put. ++ */ ++ if (limit > 0 && x > 0 && isWidecExt(str[0])) { ++ --str; ++ --x; ++ ++limit; ++ } ++ if (limit > 0 && x + limit < screen_columns(sp) && isWidecExt(str[limit])) ++ ++limit; ++ ++ { ++ MakeArray(ci, CHAR_INFO, limit); ++ ++ for (i = actual = 0; i < limit; i++) { ++ ch = str[i]; ++ if (isWidecExt(ch)) { ++ if (actual > 0 ++ && (ci[actual - 1].Attributes & COMMON_LVB_LEADING_BYTE)) { ++ ci[actual] = ci[actual - 1]; ++ ci[actual].Attributes = (WORD) ++ ((ci[actual].Attributes & ~COMMON_LVB_LEADING_BYTE) ++ | COMMON_LVB_TRAILING_BYTE); + } else { + ci[actual].CharInfoChar = ' '; ++ ci[actual].Attributes = MapAttr(WINCONSOLE.SBI.wAttributes, ++ AttrOf(ch)); + } ++ ++actual; ++ continue; + } ++ ci[actual].CharInfoChar = CharOf(ch); ++ ci[actual].Attributes = MapAttr(WINCONSOLE.SBI.wAttributes, ++ AttrOf(ch)); ++ if (AttrOf(ch) & A_ALTCHARSET) { ++ if (_nc_wacs) { ++ int which = CharOf(ch); ++ if (which > 0 ++ && which < ACS_LEN ++ && CharOf(_nc_wacs[which]) != 0) { ++ ci[actual].CharInfoChar = CharOf(_nc_wacs[which]); ++ } else { ++ ci[actual].CharInfoChar = ' '; ++ } ++ } ++ } ++ if (i + 1 < limit && isWidecExt(str[i + 1])) ++ ci[actual].Attributes |= COMMON_LVB_LEADING_BYTE; ++ ++actual; + } +- ++actual; +- } + +- loc.X = (SHORT) 0; +- loc.Y = (SHORT) 0; +- siz.X = (SHORT) actual; +- siz.Y = 1; +- +- rec.Left = (SHORT) x; +- rec.Top = (SHORT) (y + AdjustY()); +- rec.Right = (SHORT) (x + limit - 1); +- rec.Bottom = rec.Top; ++ loc.X = (SHORT) 0; ++ loc.Y = (SHORT) 0; ++ siz.X = (SHORT) actual; ++ siz.Y = 1; ++ ++ rec.Left = (SHORT) x; ++ rec.Top = (SHORT) (y + AdjustY()); ++ rec.Right = (SHORT) (x + actual - 1); ++ rec.Bottom = rec.Top; + +- return write_screen(WINCONSOLE.hdl, ci, siz, loc, &rec); ++ return write_screen(WINCONSOLE.hdl, ci, siz, loc, &rec); ++ } + } + #define con_write(tcb, y, x, str, n) con_write16(tcb, y, x, str, n) + #else diff --git a/scripts/release_windows.sh b/scripts/release_windows.sh new file mode 100755 index 0000000..b68b5ba --- /dev/null +++ b/scripts/release_windows.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Publish dist/cst-agent-VERSION-windows-x86_64.zip (+ .sha256) to the Gitea +# release vVERSION, creating the release (and its tag at HEAD) when missing. +# GITEA_TOKEN=... scripts/release_windows.sh [--build] +# --build runs scripts/bootstrap_windows.sh first. The token is only read from +# the environment and handed to curl on stdin: it never appears in argv, on +# disk or in the output. Assets of the same name are replaced. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SERVER="${CST_GITEA:-https://git.whcself.xyz}" +REPO="${CST_REPO:-WHC/CST-agent}" +API="$SERVER/api/v1/repos/$REPO" + +[[ -n "${GITEA_TOKEN:-}" ]] || { + echo "set GITEA_TOKEN (a Gitea token with write:repository)" >&2 + exit 1 +} +command -v python3 >/dev/null || { echo "missing python3" >&2; exit 1; } +cd "$ROOT" +[[ "${1:-}" == --build ]] && scripts/bootstrap_windows.sh "$ROOT" + +ver="$(sed -n 's/^#define CST_VERSION_\(MAJOR\|MINOR\|PATCH\) \([0-9]*\)$/\2/p' include/cst_version.h | paste -sd.)" +tag="v$ver" +zip="dist/cst-agent-$ver-windows-x86_64.zip" +for f in "$zip" "$zip.sha256"; do + [[ -f "$f" ]] || { echo "missing $f (scripts/package_windows.sh, or --build)" >&2; exit 1; } +done + +api() { # METHOD PATH [curl args...] + local m="$1" p="$2" + shift 2 + printf 'Authorization: token %s\n' "$GITEA_TOKEN" | + curl -fsS -X "$m" -H @- "$API$p" "$@" +} +json() { # CODE [ARGS...]: d = the JSON on stdin, ARGS in sys.argv[1:] + local code="$1" + shift + python3 -c "import json,sys; d=json.load(sys.stdin); $code" "$@" +} + +if rel="$(api GET "/releases/tags/$tag" 2>/dev/null)"; then + echo "==> release $tag exists" +else + head="$(git rev-parse HEAD)" + if ! git branch -r --contains "$head" 2>/dev/null | grep -q .; then + echo "HEAD $head is not on any remote branch; push it first" >&2 + exit 1 + fi + echo "==> creating release $tag at $head" + body="$(python3 -c 'import json,sys; print(json.dumps({"tag_name": sys.argv[1], "target_commitish": sys.argv[2], "name": "CST-Agent " + sys.argv[1], "body": "Windows x86_64: unzip, or install with scripts/install_windows.ps1"}))' "$tag" "$head")" + rel="$(api POST /releases -H 'Content-Type: application/json' -d "$body")" +fi +id="$(json 'print(d["id"])' <<<"$rel")" + +for f in "$zip" "$zip.sha256"; do + name="$(basename "$f")" + old="$(json "print(' '.join(str(a['id']) for a in d.get('assets') or [] if a['name'] == sys.argv[1]))" "$name" <<<"$rel")" + for aid in $old; do + echo "==> replacing $name" + api DELETE "/releases/$id/assets/$aid" >/dev/null + done + echo "==> upload $name" + api POST "/releases/$id/assets?name=$name" -F "attachment=@$f" >/dev/null +done +echo "$SERVER/$REPO/releases/tag/$tag" diff --git a/scripts/run_windows.bat b/scripts/run_windows.bat deleted file mode 100755 index e0c8eff..0000000 --- a/scripts/run_windows.bat +++ /dev/null @@ -1,27 +0,0 @@ -@echo off -REM Run ON Windows after copying dist/windows-x86_64 (or a full git checkout + MSYS2 build). -REM This script only documents env + launch; prefer bootstrap_windows.sh on Linux for cross-build. - -setlocal -if "%OPENAI_API_KEY%"=="" if "%CST_API_KEY%"=="" ( - echo Set OPENAI_API_KEY or CST_API_KEY first. See docs\env_openai.md - echo Example: - echo set OPENAI_API_KEY=sk-... - echo set OPENAI_BASE_URL=https://api.openai.com/v1 - echo set OPENAI_MODEL=gpt-4.1-mini - exit /b 1 -) - -if not exist cst-agent.exe ( - echo cst-agent.exe not found in %CD% - echo Copy dist\windows-x86_64\* here, or build with MSYS2/MinGW. - exit /b 1 -) - -if not exist busybox.exe ( - echo WARNING: busybox.exe missing — run_shell may fail. Place busybox.exe beside cst-agent.exe -) - -echo Launching CST-Agent... -cst-agent.exe --workspace . -endlocal diff --git a/scripts/win_test.sh b/scripts/win_test.sh new file mode 100755 index 0000000..42b8a3c --- /dev/null +++ b/scripts/win_test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Run the cross-built Windows tests inside the test VMs (scripts/winvm/vm.sh). +# +# scripts/win_test.sh [-b BUILD_DIR] [-v "en zh"] [-u "Administrator tester 测试"] [-f FILTER] +# +# BUILD_DIR (default build-win) must contain test_*.exe from the MinGW build; +# busybox.exe is taken from third_party/busybox/. VMs that are not running are +# started and stopped again afterwards. Exit status is the number of failed +# variant/user runs. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +VM="$ROOT/scripts/winvm/vm.sh" +BUILD="$ROOT/build-win" +VARIANTS="en zh" +USERS_LIST="Administrator tester 测试" +FILTER="*" +while getopts "b:v:u:f:" o; do + case "$o" in + b) BUILD="$(cd "$OPTARG" && pwd)" ;; + v) VARIANTS="$OPTARG" ;; + u) USERS_LIST="$OPTARG" ;; + f) FILTER="$OPTARG" ;; + *) sed -n '2,8p' "$0"; exit 2 ;; + esac +done + +shopt -s nullglob +exes=("$BUILD"/test_*.exe) +[[ ${#exes[@]} -gt 0 ]] || { echo "no test_*.exe in $BUILD" >&2; exit 1; } + +stage="$(mktemp -d)" +trap 'rm -rf "$stage"' EXIT +mkdir -p "$stage/bundle/bin" "$stage/bundle/data/tests" +cp "${exes[@]}" "$stage/bundle/bin/" +for f in "$BUILD"/*.dll "$ROOT/third_party/busybox/busybox.exe"; do + [[ -f "$f" ]] && cp "$f" "$stage/bundle/bin/" +done +cp -r "$ROOT/tests/fixtures" "$stage/bundle/data/tests/" +cp "$ROOT/scripts/winvm/run_tests.ps1" "$stage/bundle/" +tar -C "$stage" -cf "$stage/bundle.tar" bundle + +total_fail=0 +summary=() +for v in $VARIANTS; do + started=0 + if [[ "$("$VM" status "$v")" != running ]]; then + "$VM" start "$v" + started=1 + fi + "$VM" ssh "$v" 'Remove-Item -Recurse -Force C:\cst-test\bundle -ErrorAction SilentlyContinue; New-Item -ItemType Directory -Force C:\cst-test | Out-Null' >/dev/null + "$VM" put "$v" "$stage/bundle.tar" "C:/cst-test/bundle.tar" + "$VM" ssh "$v" 'tar -xf C:\cst-test\bundle.tar -C C:\cst-test; Remove-Item C:\cst-test\bundle.tar' >/dev/null + for u in $USERS_LIST; do + echo "===== $v / $u" + rc=0 + "$VM" ps "$v" "$u" "$stage/bundle/run_tests.ps1" -Bundle 'C:\cst-test\bundle' -Filter "'$FILTER'" || rc=$? + summary+=("$v/$u: $([[ $rc == 0 ]] && echo ok || echo FAIL)") + [[ $rc == 0 ]] || total_fail=$((total_fail + 1)) + done + [[ $started == 1 ]] && "$VM" stop "$v" +done + +echo +printf '%s\n' "${summary[@]}" +exit "$total_fail" diff --git a/scripts/windows/cst-agent-update.cmd b/scripts/windows/cst-agent-update.cmd new file mode 100644 index 0000000..f8f84bd --- /dev/null +++ b/scripts/windows/cst-agent-update.cmd @@ -0,0 +1,4 @@ +@echo off +rem Update CST-Agent in this folder from the latest release. +rem cst-agent-update [-Version 0.6.0] +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0install_windows.ps1" -Prefix "%~dp0." %* diff --git a/scripts/winvm/autounattend.xml.in b/scripts/winvm/autounattend.xml.in new file mode 100644 index 0000000..8dead76 --- /dev/null +++ b/scripts/winvm/autounattend.xml.in @@ -0,0 +1,98 @@ + + + + + + en-US + + en-US + en-US + en-US + en-US + + + + + 0 + true + + + 1 + Primary + true + + + + + 1 + 1 + true + NTFS + + C + + + + + + + + + /IMAGE/NAME + @IMAGE_NAME@ + + + + 0 + 1 + + + + + true + CST + CST + + + + + + cst-win + China Standard Time + + + + + + true + true + true + 3 + Work + + + + @ADMIN_PASSWORD@ + true</PlainText> + </AdministratorPassword> + </UserAccounts> + <AutoLogon> + <Enabled>true</Enabled> + <Username>Administrator</Username> + <Password> + <Value>@ADMIN_PASSWORD@</Value> + <PlainText>true</PlainText> + </Password> + <LogonCount>1</LogonCount> + </AutoLogon> + <FirstLogonCommands> + <SynchronousCommand wcm:action="add"> + <Order>1</Order> + <CommandLine>powershell -NoProfile -ExecutionPolicy Bypass -Command "foreach ($d in Get-PSDrive -PSProvider FileSystem) { $p = Join-Path $d.Root 'setup.ps1'; if (Test-Path $p) { &amp; $p; break } }"</CommandLine> + <Description>Enable OpenSSH</Description> + <RequiresUserInput>false</RequiresUserInput> + </SynchronousCommand> + </FirstLogonCommands> + </component> + </settings> +</unattend> diff --git a/scripts/winvm/provision.ps1 b/scripts/winvm/provision.ps1 new file mode 100644 index 0000000..7d94151 --- /dev/null +++ b/scripts/winvm/provision.ps1 @@ -0,0 +1,28 @@ +# Idempotent VM setup after install: test users, sshd access, console defaults. +# Users come from C:\Windows\Temp\cst-users.txt ("name<TAB>password", UTF-8), deleted afterwards. +$ErrorActionPreference = 'Stop' + +$list = 'C:\Windows\Temp\cst-users.txt' +if (Test-Path $list) { + foreach ($line in Get-Content -Path $list -Encoding UTF8) { + $name, $pw = $line -split "`t", 2 + if (-not $name) { continue } + $sec = ConvertTo-SecureString $pw -AsPlainText -Force + if (Get-LocalUser -Name $name -ErrorAction SilentlyContinue) { + Set-LocalUser -Name $name -Password $sec + } else { + New-LocalUser -Name $name -Password $sec -PasswordNeverExpires -AccountNeverExpires | Out-Null + } + foreach ($g in 'Users', 'OpenSSH Users') { + Add-LocalGroupMember -Group $g -Member $name -ErrorAction SilentlyContinue + } + "user ok: $name" + } + Remove-Item -Force $list +} + +# the built-in rule only covers the Private profile; QEMU user networking is Public +Set-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -Profile Any -Enabled True +Set-SConfig -AutoLaunch $false +New-Item -ItemType Directory -Force C:\cst-test | Out-Null +"provision ok" diff --git a/scripts/winvm/run_tests.ps1 b/scripts/winvm/run_tests.ps1 new file mode 100644 index 0000000..9fc42c6 --- /dev/null +++ b/scripts/winvm/run_tests.ps1 @@ -0,0 +1,51 @@ +# Run cross-built test_*.exe inside the VM as the current user. +# run_tests.ps1 BUNDLE [FILTER] +# BUNDLE holds bin\ (exes, busybox.exe) and data\ (repo-relative fixtures). +# Each run copies data\ into a per-user work dir and uses it as the cwd. +param([Parameter(Mandatory = $true)][string]$Bundle, [string]$Filter = '*') +$ErrorActionPreference = 'Stop' +[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$OutputEncoding = [Console]::OutputEncoding + +$work = Join-Path $env:TEMP ('cst-test-' + [guid]::NewGuid().ToString('N').Substring(0, 8)) +New-Item -ItemType Directory -Force $work | Out-Null +Copy-Item -Recurse -Force (Join-Path $Bundle 'data\*') $work +$env:PATH = (Join-Path $Bundle 'bin') + ';' + $env:PATH + +$acp = [Text.Encoding]::Default.CodePage +"== user=$env:USERNAME acp=$acp work=$work" +$fail = 0 +$pass = 0 +foreach ($exe in Get-ChildItem (Join-Path $Bundle 'bin') -Filter "test_$Filter.exe" | Sort-Object Name) { + $name = $exe.BaseName + $out = Join-Path $work "$name.out" + $err = Join-Path $work "$name.err" + $sw = [Diagnostics.Stopwatch]::StartNew() + $p = Start-Process -FilePath $exe.FullName -WorkingDirectory $work -NoNewWindow -PassThru ` + -RedirectStandardOutput $out -RedirectStandardError $err + $null = $p.Handle + if (-not $p.WaitForExit(300000)) { + & taskkill.exe /T /F /PID $p.Id | Out-Null + $code = 'timeout' + } else { + $p.WaitForExit() + $code = $p.ExitCode + } + $secs = [math]::Round($sw.Elapsed.TotalSeconds, 1) + if ($code -eq 0) { + $pass++ + "PASS $name (${secs}s)" + } else { + $fail++ + "FAIL $name (exit $code, ${secs}s)" + foreach ($f in $out, $err) { + if (Test-Path $f) { + Get-Content -Path $f -Encoding UTF8 -Tail 25 | ForEach-Object { " $_" } + } + } + } +} +"== $pass passed, $fail failed" +Set-Location $env:TEMP +Remove-Item -Recurse -Force $work -ErrorAction SilentlyContinue +exit $fail diff --git a/scripts/winvm/setup.ps1 b/scripts/winvm/setup.ps1 new file mode 100644 index 0000000..e7e42d9 --- /dev/null +++ b/scripts/winvm/setup.ps1 @@ -0,0 +1,28 @@ +Start-Transcript -Path C:\setup.log -Append +$ErrorActionPreference = 'Continue' + +if (-not (Get-Service sshd -ErrorAction SilentlyContinue)) { + Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +} +Set-Service -Name sshd -StartupType Automatic +Start-Service sshd + +$keys = 'C:\ProgramData\ssh\administrators_authorized_keys' +Get-Content (Join-Path $PSScriptRoot 'vm_key.pub') | Set-Content -Path $keys -Encoding ascii +icacls $keys /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F' + +New-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH -Name DefaultShell -PropertyType String -Force ` + -Value 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' + +if (Get-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -ErrorAction SilentlyContinue) { + # the built-in rule only covers the Private profile; QEMU user networking is Public + Set-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -Profile Any -Enabled True +} else { + New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' ` + -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -Profile Any +} + +powercfg /change monitor-timeout-ac 0 +powercfg /change standby-timeout-ac 0 +New-Item -Path C:\setup.done -ItemType File -Force +Stop-Transcript diff --git a/scripts/winvm/type.py b/scripts/winvm/type.py new file mode 100644 index 0000000..f2fe1f8 --- /dev/null +++ b/scripts/winvm/type.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Type text into the VM console via QMP. + + type.py QMP_SOCKET TEXT (\\n = Enter) + type.py QMP_SOCKET @FILE the file's first line, then Enter (secrets stay + off the command line) + type.py QMP_SOCKET --cad Ctrl+Alt+Del +""" +import json +import os +import socket +import sys +import time + +SHIFTED = { + '!': '1', '@': '2', '#': '3', '$': '4', '%': '5', '^': '6', '&': '7', + '*': '8', '(': '9', ')': '0', '_': 'minus', '+': 'equal', '{': 'bracket_left', + '}': 'bracket_right', '|': 'backslash', ':': 'semicolon', '"': 'apostrophe', + '<': 'comma', '>': 'dot', '?': 'slash', '~': 'grave_accent', +} +PLAIN = { + ' ': 'spc', '\n': 'ret', '\t': 'tab', '-': 'minus', '=': 'equal', + '[': 'bracket_left', ']': 'bracket_right', '\\': 'backslash', ';': 'semicolon', + "'": 'apostrophe', ',': 'comma', '.': 'dot', '/': 'slash', '`': 'grave_accent', +} + + +def keys_for(ch): + if ch.isascii() and ch.isalpha(): + return (['shift'] if ch.isupper() else []) + [ch.lower()] + if ch.isdigit(): + return [ch] + if ch in PLAIN: + return [PLAIN[ch]] + if ch in SHIFTED: + return ['shift', SHIFTED[ch]] + raise SystemExit(f'unsupported char {ch!r}') + + +def main(): + arg = sys.argv[2] + if arg == '--cad': + combos = [['ctrl', 'alt', 'delete']] + else: + if arg.startswith('@'): + with open(arg[1:], encoding='utf-8') as fp: + text = fp.readline().rstrip('\r\n') + '\n' + else: + text = arg.encode().decode('unicode_escape') + combos = [keys_for(ch) for ch in text] + s = socket.socket(socket.AF_UNIX) + s.connect(sys.argv[1]) + f = s.makefile('rw') + f.readline() + f.write(json.dumps({'execute': 'qmp_capabilities'}) + '\n') + f.flush() + f.readline() + for combo in combos: + keys = [{'type': 'qcode', 'data': k} for k in combo] + f.write(json.dumps({'execute': 'send-key', 'arguments': {'keys': keys}}) + '\n') + f.flush() + f.readline() + time.sleep(0.03) + + +main() diff --git a/scripts/winvm/vm.sh b/scripts/winvm/vm.sh new file mode 100755 index 0000000..d8ee115 --- /dev/null +++ b/scripts/winvm/vm.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +# Headless Windows Server 2025 Core test VMs (QEMU/KVM). +# +# vm.sh create download the evaluation ISO, install base.qcow2 +# vm.sh variants en.qcow2 / zh.qcow2 overlays on top of base.qcow2 +# (zh: zh-CN system locale, ANSI code page 936) +# vm.sh start|stop|status|wait VARIANT +# vm.sh ssh VARIANT [USER] [command...] USER: Administrator (default), tester, 测试 +# vm.sh put VARIANT SRC... DEST scp as Administrator +# vm.sh ps VARIANT [USER] FILE.ps1 [args...] run a script (uploaded with a UTF-8 BOM) +# vm.sh shot VARIANT [out.png] console screenshot +# vm.sh type VARIANT TEXT type into the console (\n = Enter) +# vm.sh login VARIANT unlock the console as Administrator +# +# State (ISO, disks, keys, passwords) lives in $CST_WINVM_DIR, default ~/winvm. +# Keys and passwords are created there with mode 0600 and never printed. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +D="${CST_WINVM_DIR:-$HOME/winvm}" +ISO_URL="https://aka.ms/WinServ2025iso-enus" +IMAGE_NAME="Windows Server 2025 SERVERSTANDARDCORE" +USERS=(tester 测试) + +if ! command -v qemu-system-x86_64 >/dev/null || ! command -v sshpass >/dev/null || + ! command -v socat >/dev/null || ! command -v xorriso >/dev/null; then + if command -v nix-shell >/dev/null && [[ -z "${CST_WINVM_NIX:-}" ]]; then + export CST_WINVM_NIX=1 + exec nix-shell -p qemu sshpass socat xorriso imagemagick python3 \ + --run "$(printf '%q ' "$0" "$@")" + fi + echo "need qemu-system-x86_64, sshpass, socat, xorriso" >&2 + exit 1 +fi + +die() { echo "vm.sh: $*" >&2; exit 1; } + +port_of() { + case "$1" in + base) echo 2221 ;; + en) echo 2222 ;; + zh) echo 2223 ;; + *) die "unknown variant '$1' (base, en, zh)" ;; + esac +} + +pw_file() { + case "$1" in + Administrator) echo "$D/admin-password" ;; + tester) echo "$D/tester-password" ;; + 测试) echo "$D/ceshi-password" ;; + *) die "unknown user '$1'" ;; + esac +} + +new_password() { # prefix -> file, mixed classes for Windows complexity rules + local f="$2" + [[ -s "$f" ]] && return 0 + (umask 077; printf '%s%s9a' "$1" "$(head -c 256 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 16)" >"$f") +} + +running() { + local pid + pid="$(cat "$D/$1.pid" 2>/dev/null)" || return 1 + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null +} + +ssh_opts() { + echo -p "$(port_of "$1")" -o UserKnownHostsFile="$D/known_hosts" \ + -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR -o ConnectTimeout=10 +} + +vm_ssh() { # variant user command... + local v="$1" u="$2" + shift 2 + # shellcheck disable=SC2046 + if [[ "$u" == Administrator ]]; then + ssh -i "$D/vm_key" -o BatchMode=yes $(ssh_opts "$v") "Administrator@127.0.0.1" "$@" + else + SSHPASS="$(cat "$(pw_file "$u")")" sshpass -e ssh -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password $(ssh_opts "$v") "$u@127.0.0.1" "$@" + fi +} + +vm_put() { # variant src... dest + local v="$1" + shift + local dest="${!#}" + local srcs=("${@:1:$#-1}") + # shellcheck disable=SC2046 + scp -q -r -i "$D/vm_key" -o BatchMode=yes $(ssh_opts "$v" | sed 's/^-p/-P/') \ + "${srcs[@]}" "Administrator@127.0.0.1:$dest" +} + +qemu_start() { # variant disk [extra args...] + local v="$1" disk="$2" + shift 2 + running "$v" && return 0 + [[ -f "$disk" ]] || die "missing $disk" + qemu-system-x86_64 \ + -name "cst-win-$v" -machine q35,accel=kvm \ + -cpu host,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time \ + -smp 4 -m 6G -rtc base=localtime \ + -device ahci,id=ahci \ + -drive file="$disk",if=none,id=d0,cache=writeback,discard=unmap \ + -device ide-hd,drive=d0,bus=ahci.0 \ + -netdev user,id=n0,hostfwd=tcp:127.0.0.1:"$(port_of "$v")"-:22 \ + -device e1000e,netdev=n0 \ + -device qemu-xhci,id=xhci -device usb-tablet,bus=xhci.0 \ + -display none -qmp unix:"$D/$v.qmp",server,nowait \ + -pidfile "$D/$v.pid" -daemonize "$@" +} + +disk_of() { + case "$1" in + base) echo "$D/base.qcow2" ;; + *) echo "$D/$1.qcow2" ;; + esac +} + +vm_wait() { # variant [timeout_s] + local v="$1" t="${2:-900}" i + for ((i = 0; i < t; i += 5)); do + if vm_ssh "$v" Administrator 'Test-Path C:\setup.done' 2>/dev/null | grep -q True; then + return 0 + fi + running "$v" || die "$v is not running" + sleep 5 + done + die "$v: no SSH after ${t}s" +} + +vm_stop() { + local v="$1" i + running "$v" || return 0 + vm_ssh "$v" Administrator 'Stop-Computer -Force' >/dev/null 2>&1 || true + for ((i = 0; i < 120; i++)); do + running "$v" || { rm -f "$D/$v.pid" "$D/$v.qmp"; return 0; } + sleep 1 + done + printf '%s\n' '{"execute":"qmp_capabilities"}' '{"execute":"quit"}' | + socat - UNIX-CONNECT:"$D/$v.qmp" >/dev/null || true + rm -f "$D/$v.pid" "$D/$v.qmp" +} + +with_bom() { # file -> temp copy with a UTF-8 BOM (Windows PowerShell 5.1 reads ANSI otherwise) + local out + out="$(mktemp --suffix=.ps1)" + printf '\xef\xbb\xbf' >"$out" + sed '1s/^\xef\xbb\xbf//' "$1" >>"$out" + echo "$out" +} + +vm_ps() { # variant user file args... + local v="$1" u="$2" f="$3" + shift 3 + local tmp name + tmp="$(with_bom "$f")" + name="cst-$(basename "$f" .ps1)-$$.ps1" + vm_put "$v" "$tmp" "C:/Windows/Temp/$name" + rm -f "$tmp" + vm_ssh "$v" Administrator "icacls C:\\Windows\\Temp\\$name /grant 'Users:RX' | Out-Null" >/dev/null + local rc=0 + vm_ssh "$v" "$u" "powershell -NoProfile -ExecutionPolicy Bypass -File C:\\Windows\\Temp\\$name $*" || rc=$? + vm_ssh "$v" Administrator "Remove-Item -Force C:\\Windows\\Temp\\$name" >/dev/null 2>&1 || true + return $rc +} + +provision() { # variant: users, sshd, console; idempotent + local v="$1" u tmp + tmp="$(mktemp)" + chmod 600 "$tmp" + for u in "${USERS[@]}"; do + new_password "Tst!" "$(pw_file "$u")" + printf '%s\t%s\n' "$u" "$(cat "$(pw_file "$u")")" >>"$tmp" + done + vm_put "$v" "$tmp" "C:/Windows/Temp/cst-users.txt" + rm -f "$tmp" + vm_ps "$v" Administrator "$HERE/provision.ps1" +} + +cmd_create() { + mkdir -p "$D" + chmod 700 "$D" + [[ -f "$D/base.qcow2" ]] && die "$D/base.qcow2 exists" + if [[ ! -s "$D/server2025.iso" ]]; then + echo "==> downloading the Windows Server 2025 evaluation ISO (~8 GB)" + curl -fL -C - -o "$D/server2025.iso" "$ISO_URL" + fi + [[ -f "$D/vm_key" ]] || ssh-keygen -q -t ed25519 -N "" -C cst-winvm -f "$D/vm_key" + new_password "Cst!" "$D/admin-password" + + local u + u="$(mktemp -d)" + chmod 700 "$u" + cp "$HERE/setup.ps1" "$D/vm_key.pub" "$u/" + (umask 077 + sed -e "s|@IMAGE_NAME@|$IMAGE_NAME|" -e "s|@ADMIN_PASSWORD@|$(cat "$D/admin-password")|g" \ + "$HERE/autounattend.xml.in" >"$u/autounattend.xml" + xorriso -as mkisofs -quiet -J -r -V UNATTEND -o "$D/unattend.iso" "$u/" 2>/dev/null) + rm -rf "$u" + + qemu-img create -q -f qcow2 "$D/base.qcow2" 60G + echo "==> unattended install (20-30 minutes)" + qemu_start base "$D/base.qcow2" \ + -drive file="$D/server2025.iso",media=cdrom,if=none,id=cd0,readonly=on \ + -device ide-cd,drive=cd0,bus=ahci.1 \ + -drive file="$D/unattend.iso",media=cdrom,if=none,id=cd1,readonly=on \ + -device ide-cd,drive=cd1,bus=ahci.2 \ + -boot order=c,once=d + vm_wait base 3600 + provision base + vm_stop base + echo "OK: $D/base.qcow2; next: vm.sh variants" +} + +cmd_variants() { + [[ -f "$D/base.qcow2" ]] || die "no base.qcow2; run vm.sh create" + running base && die "stop the base VM first" + local v + for v in en zh; do + running "$v" && vm_stop "$v" + rm -f "$D/$v.qcow2" + qemu-img create -q -f qcow2 -b "$D/base.qcow2" -F qcow2 "$D/$v.qcow2" + done + qemu_start zh "$D/zh.qcow2" + vm_wait zh + vm_ssh zh Administrator 'Set-WinSystemLocale -SystemLocale zh-CN; Set-Culture zh-CN; Set-WinHomeLocation -GeoId 45; Restart-Computer -Force' || true + sleep 20 + vm_wait zh + echo "zh ANSI code page: $(vm_ssh zh Administrator '[Text.Encoding]::Default.CodePage')" + vm_stop zh + echo "OK: en.qcow2 zh.qcow2" +} + +cmd="${1:-}" +shift || true +case "$cmd" in +create) cmd_create ;; +variants) cmd_variants ;; +provision) provision "${1:?variant}" ;; +start) + qemu_start "${1:?variant}" "$(disk_of "$1")" + vm_wait "$1" + ;; +stop) vm_stop "${1:?variant}" ;; +status) running "${1:?variant}" && echo running || echo stopped ;; +wait) vm_wait "${1:?variant}" "${2:-900}" ;; +ssh) + v="${1:?variant}" + shift + u=Administrator + if [[ $# -gt 0 ]] && [[ "$1" == Administrator || "$1" == tester || "$1" == 测试 ]]; then + u="$1" + shift + fi + vm_ssh "$v" "$u" "$@" + ;; +put) vm_put "$@" ;; +ps) + v="${1:?variant}" + shift + u=Administrator + if [[ "$1" == Administrator || "$1" == tester || "$1" == 测试 ]]; then + u="$1" + shift + fi + vm_ps "$v" "$u" "$@" + ;; +shot) + v="${1:?variant}" + out="${2:-$D/$v.png}" + printf '%s\n' '{"execute":"qmp_capabilities"}' \ + '{"execute":"screendump","arguments":{"filename":"'"$D/$v.ppm"'"}}' | + socat - UNIX-CONNECT:"$D/$v.qmp" >/dev/null + sleep 1 + magick "$D/$v.ppm" -resize '1024x1024>' "$out" + rm -f "$D/$v.ppm" + echo "$out" + ;; +type) python3 "$HERE/type.py" "$D/${1:?variant}.qmp" "${2:?text}" ;; +login) + python3 "$HERE/type.py" "$D/${1:?variant}.qmp" --cad + sleep 3 + python3 "$HERE/type.py" "$D/$1.qmp" "@$D/admin-password" + ;; +*) sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; +esac diff --git a/shell-win.nix b/shell-win.nix new file mode 100644 index 0000000..44d09b1 --- /dev/null +++ b/shell-win.nix @@ -0,0 +1,24 @@ +# Windows x86_64 cross toolchain (mingw-w64, UCRT runtime, static winpthreads). +# nix-shell shell-win.nix --run 'scripts/bootstrap_windows.sh' +{ pkgs ? import <nixpkgs> { } }: +let + win = pkgs.pkgsCross.ucrt64; +in +win.mkShell { + nativeBuildInputs = with pkgs; [ + cmake + ninja + gnumake + git + perl + python3 + pkg-config + zip + curl + ]; + buildInputs = [ win.windows.pthreads ]; + # mkShell exports CC/AR for the target; the build scripts call the prefixed tools directly. + shellHook = '' + export MINGW_HOST=x86_64-w64-mingw32 + ''; +} diff --git a/src/agent_loop.c b/src/agent_loop.c index da8bc6c..0f41946 100644 --- a/src/agent_loop.c +++ b/src/agent_loop.c @@ -270,7 +270,7 @@ static void git_branch(const char *ws, char *out, size_t n) { for (int depth = 0; depth < 8; depth++) { char p[4200], line[512]; snprintf(p, sizeof(p), "%s/.git", dir); - FILE *fp = fopen(p, "r"); + FILE *fp = fopen(p, "rb"); if (fp) { /* worktree / submodule: ".git" is a file "gitdir: <path>" */ char gd[4096] = ""; @@ -286,7 +286,7 @@ static void git_branch(const char *ws, char *out, size_t n) { snprintf(p, sizeof(p), "%s/HEAD", gd); else snprintf(p, sizeof(p), "%s/.git/HEAD", dir); - fp = fopen(p, "r"); + fp = fopen(p, "rb"); if (fp) { if (fgets(line, sizeof(line), fp)) { line[strcspn(line, "\r\n")] = 0; @@ -308,7 +308,13 @@ static void git_branch(const char *ws, char *out, size_t n) { static void os_label(char *out, size_t n) { #ifdef _WIN32 - snprintf(out, n, "Windows"); + char ver[64]; + os_windows_version(ver, sizeof(ver)); + snprintf(out, n, + "%s; run_shell is busybox sh (POSIX syntax and applets; write paths " + "as C:/dir or relative, Windows programs like git or python run " + "from PATH)", + ver); #else struct utsname u; if (uname(&u) == 0) @@ -780,9 +786,14 @@ static void sandbox_escalate(turn_t *t, cst_tool_ctx_t *ctx, free(detail); } if (ans == CST_PERM_DENY) { - const char *note = "\n[cst: blocked by the sandbox (writes are limited to " - "the workspace and temp dir, no network); the user did " - "not allow running it outside]\n"; + const char *note = + cst_sandbox_probe() == CST_SANDBOX_LOWIL + ? "\n[cst: blocked by the sandbox (writes are limited to the " + "workspace and temp dir); the user did not allow running it " + "outside]\n" + : "\n[cst: blocked by the sandbox (writes are limited to " + "the workspace and temp dir, no network); the user did " + "not allow running it outside]\n"; cst_buf_t b = {0}; if (tr->out) cst_buf_append(&b, tr->out, tr->out_len); @@ -924,9 +935,11 @@ static void exec_begin(turn_t *t, pcall_t *pc, exec_t *x) { if (def->perm == CST_TP_WRITE) { ask = 0; } else if (def->perm == CST_TP_SHELL) { - if (cst_sandbox_probe() != CST_SANDBOX_NONE) { + int sb = cst_sandbox_probe(); + if (sb != CST_SANDBOX_NONE) { ctx->sandbox = 1; - ask = 0; + /* the Low-IL sandbox does not stop network access */ + ask = sb == CST_SANDBOX_LOWIL && cmd && cst_shell_uses_network(cmd); } else if (cmd && cst_shell_is_safe(cmd)) { ask = 0; } @@ -1707,8 +1720,12 @@ void cst_agent_session_id(const cst_agent_t *a, char *out, size_t out_sz) { return; const char *p = a->cfg.session_path; const char *sl = strrchr(p, '/'); - snprintf(out, out_sz, "%s", sl ? sl + 1 : p); - size_t n = strlen(out); + const char *base = sl ? sl + 1 : p; + size_t n = strlen(base); + if (n >= out_sz) + n = out_sz - 1; + memcpy(out, base, n); + out[n] = 0; if (n > 6 && strcmp(out + n - 6, ".jsonl") == 0) out[n - 6] = 0; } @@ -1747,7 +1764,8 @@ void cst_agent_reload_notes(cst_agent_t *a) { fclose(fp); if (b.len && b.buf) { size_t clean_len = 0; - a->notes = cst_utf8_sanitize(b.buf, b.len, &clean_len); + size_t bom = cst_bom_len(b.buf, b.len); + a->notes = cst_utf8_sanitize(b.buf + bom, b.len - bom, &clean_len); } cst_buf_free(&b); return; diff --git a/src/agent_store.c b/src/agent_store.c index 57d4086..23628d4 100644 --- a/src/agent_store.c +++ b/src/agent_store.c @@ -6,7 +6,6 @@ #include "cst_agent.h" #include "os_compat.h" -#include <dirent.h> #include <errno.h> #include <stdio.h> #include <stdlib.h> @@ -45,7 +44,7 @@ int cst_agent_store_ensure(const char *workspace) { char gi[OS_PATH_MAX]; if (os_path_join(gi, sizeof(gi), base, ".gitignore") == 0 && OS_ACCESS(gi, OS_F_OK) != 0) { - FILE *fp = fopen(gi, "w"); + FILE *fp = fopen(gi, "wb"); if (fp) { fputs("*\n", fp); fclose(fp); @@ -97,7 +96,7 @@ int cst_agent_store_set_current(const char *workspace, const char *id) { return -1; if (os_path_join(path, sizeof(path), dir, "CURRENT") != 0) return -1; - FILE *fp = fopen(path, "w"); + FILE *fp = fopen(path, "wb"); if (!fp) return -1; fprintf(fp, "%s\n", id); @@ -113,7 +112,7 @@ int cst_agent_store_get_current(const char *workspace, char *id_out, return -1; if (os_path_join(path, sizeof(path), dir, "CURRENT") != 0) return -1; - FILE *fp = fopen(path, "r"); + FILE *fp = fopen(path, "rb"); if (!fp) return -1; if (!fgets(id_out, (int)id_sz, fp)) { @@ -159,7 +158,7 @@ int cst_agent_store_list(const char *workspace, char ids[][128], int max_ids) { char dir[OS_PATH_MAX]; if (cst_agent_store_dir(workspace, dir, sizeof(dir)) != 0) return 0; - DIR *d = opendir(dir); + os_dir_t *d = os_dir_open(dir); if (!d) return 0; typedef struct { @@ -168,23 +167,23 @@ int cst_agent_store_list(const char *workspace, char ids[][128], int max_ids) { } ent_t; ent_t *ents = calloc((size_t)max_ids * 4, sizeof(ent_t)); int n = 0, cap = max_ids * 4; - struct dirent *de; - while ((de = readdir(d)) != NULL) { - size_t len = strlen(de->d_name); - if (len < 7 || strcmp(de->d_name + len - 6, ".jsonl") != 0) + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { + size_t len = strlen(de->name); + if (len < 7 || strcmp(de->name + len - 6, ".jsonl") != 0) continue; if (n >= cap) break; snprintf(ents[n].id, sizeof(ents[n].id), "%.*s", (int)(len - 6), - de->d_name); + de->name); char full[OS_PATH_MAX]; - if (os_path_join(full, sizeof(full), dir, de->d_name) != 0) + if (os_path_join(full, sizeof(full), dir, de->name) != 0) continue; struct stat st; ents[n].mtime = (stat(full, &st) == 0) ? st.st_mtime : 0; n++; } - closedir(d); + os_dir_close(d); for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { if (ents[j].mtime > ents[i].mtime) { @@ -218,7 +217,7 @@ int cst_agent_store_title_set(const char *workspace, const char *id, return -1; if (!title || !title[0]) return remove(p) == 0 || errno == ENOENT ? 0 : -1; - FILE *fp = fopen(p, "w"); + FILE *fp = fopen(p, "wb"); if (!fp) return -1; for (const char *c = title; *c; c++) @@ -255,7 +254,7 @@ int cst_agent_store_info(const char *workspace, const char *id, info->mtime = (long long)st.st_mtime; char tp[OS_PATH_MAX]; if (side_path(workspace, id, ".title", tp, sizeof(tp)) == 0) { - FILE *tf = fopen(tp, "r"); + FILE *tf = fopen(tp, "rb"); if (tf) { if (fgets(info->title, sizeof(info->title), tf)) { size_t n = strlen(info->title); @@ -265,7 +264,7 @@ int cst_agent_store_info(const char *workspace, const char *id, fclose(tf); } } - FILE *fp = fopen(p, "r"); + FILE *fp = fopen(p, "rb"); if (!fp) return 0; size_t cap = 1 << 16; @@ -311,16 +310,16 @@ int cst_agent_store_delete(const char *workspace, const char *id) { remove(p); /* sub-agent transcripts live in <id>/ */ if (side_path(workspace, id, "", p, sizeof(p)) == 0) { - DIR *d = opendir(p); + os_dir_t *d = os_dir_open(p); if (d) { - struct dirent *de; - while ((de = readdir(d)) != NULL) { + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { char f[OS_PATH_MAX]; - if (strncmp(de->d_name, "sub-", 4) == 0 && - os_path_join(f, sizeof(f), p, de->d_name) == 0) + if (strncmp(de->name, "sub-", 4) == 0 && + os_path_join(f, sizeof(f), p, de->name) == 0) remove(f); } - closedir(d); + os_dir_close(d); rmdir(p); } } diff --git a/src/creds.c b/src/creds.c index 36f4318..7a0098f 100644 --- a/src/creds.c +++ b/src/creds.c @@ -12,6 +12,7 @@ #ifdef _WIN32 #include <io.h> +#include <windows.h> #else #include <fcntl.h> #include <termios.h> @@ -51,7 +52,7 @@ int cst_creds_load(const char *provider, char *out, size_t out_sz) { char path[OS_PATH_MAX]; if (credentials_path(path, sizeof(path)) != 0) return -1; - FILE *fp = fopen(path, "r"); + FILE *fp = fopen(path, "rb"); if (!fp) return -1; char want[64]; @@ -104,7 +105,9 @@ int cst_creds_save(const char *provider, const char *key) { size_t wl = strlen(want); #ifdef _WIN32 - FILE *out = fopen(tmp, "w"); + FILE *out = fopen(tmp, "wb"); + if (out) + (void)os_chmod_private(tmp); /* label before the key is written */ #else /* never exists world-readable, even briefly */ int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600); @@ -114,7 +117,7 @@ int cst_creds_save(const char *provider, const char *key) { #endif if (!out) return -1; - FILE *in = fopen(path, "r"); + FILE *in = fopen(path, "rb"); if (in) { char line[1024]; while (fgets(line, sizeof(line), in)) { @@ -128,10 +131,7 @@ int cst_creds_save(const char *provider, const char *key) { remove(tmp); return -1; } -#ifdef _WIN32 - remove(path); -#endif - if (rename(tmp, path) != 0) { + if (os_rename_replace(tmp, path) != 0) { remove(tmp); return -1; } @@ -167,7 +167,13 @@ int cst_creds_prompt_console(const char *provider, char *out, size_t out_sz) { fprintf(stderr, "API key for %s (input hidden, saved to the config dir): ", provider ? provider : "?"); fflush(stderr); -#ifndef _WIN32 +#ifdef _WIN32 + HANDLE hin = GetStdHandle(STD_INPUT_HANDLE); + DWORD old_mode = 0; + int tty = GetConsoleMode(hin, &old_mode) != 0; + if (tty) + SetConsoleMode(hin, old_mode & ~(DWORD)ENABLE_ECHO_INPUT); +#else struct termios old, quiet; int tty = isatty(STDIN_FILENO) && tcgetattr(STDIN_FILENO, &old) == 0; if (tty) { @@ -177,12 +183,14 @@ int cst_creds_prompt_console(const char *provider, char *out, size_t out_sz) { } #endif char *got = fgets(out, (int)out_sz, stdin); -#ifndef _WIN32 if (tty) { +#ifdef _WIN32 + SetConsoleMode(hin, old_mode); +#else tcsetattr(STDIN_FILENO, TCSAFLUSH, &old); +#endif fputc('\n', stderr); } -#endif if (!got) { out[0] = 0; return -1; diff --git a/src/keymap.c b/src/keymap.c index a62294b..72833e8 100644 --- a/src/keymap.c +++ b/src/keymap.c @@ -1,6 +1,7 @@ #include "cst_keymap.h" #include "cJSON.h" +#include "cst_util.h" #include "os_compat.h" #include <ctype.h> @@ -491,7 +492,7 @@ int cst_keymap_load(cst_keymap_t *km, char *err, size_t err_sz) { size_t n = fread(buf, 1, sizeof(buf) - 1, fp); fclose(fp); buf[n] = 0; - return cst_keymap_apply_json(km, buf, err, err_sz); + return cst_keymap_apply_json(km, buf + cst_bom_len(buf, n), err, err_sz); } int cst_keymap_lookup(const cst_keymap_t *km, const int *seq, int n, diff --git a/src/llm_openai.c b/src/llm_openai.c index 84d896d..c2028e7 100644 --- a/src/llm_openai.c +++ b/src/llm_openai.c @@ -805,11 +805,11 @@ int cst_llm_list_models(const char *api_base, const char *api_key, } CURL *curl = curl_easy_init(); if (!curl) { - snprintf(err, err_sz, "curl init failed"); - return -1; - } - llm_apply_ca_bundle(curl); - char url[1024]; + snprintf(err, err_sz, "curl init failed"); + return -1; + } + llm_apply_ca_bundle(curl); + char url[1024]; size_t bl = strlen(api_base); while (bl && api_base[bl - 1] == '/') bl--; diff --git a/src/main.c b/src/main.c index b228a6e..5c59578 100644 --- a/src/main.c +++ b/src/main.c @@ -188,6 +188,9 @@ static int demo_ctl_impl(const char *workspace) { } int main(int argc, char **argv) { + if (os_console_init() != 0) + fprintf(stderr, "warning: the process code page is not UTF-8 (Windows 10 " + "1903 or later is required); non-ASCII text may break\n"); os_set_prog_path(argv[0]); curl_global_init(CURL_GLOBAL_DEFAULT); diff --git a/src/mcp.c b/src/mcp.c index 0f22045..cef22df 100644 --- a/src/mcp.c +++ b/src/mcp.c @@ -268,6 +268,7 @@ static char *slurp(const char *path, size_t max) { fclose(fp); if (!b.buf) cst_buf_puts(&b, ""); + cst_strip_bom(b.buf, &b.len); return cst_buf_steal(&b); } diff --git a/src/metrics.c b/src/metrics.c index 39820c5..34fde34 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -47,7 +47,7 @@ cst_metrics_t *cst_metrics_open(const char *workspace, const char *session_path) snprintf(file, sizeof(file), "%s.jsonl", leaf); os_path_join(m->path, sizeof(m->path), dir, file); } - m->fp = fopen(m->path, "a"); + m->fp = fopen(m->path, "ab"); if (!m->fp) { free(m); return NULL; diff --git a/src/os_compat.c b/src/os_compat.c index a579117..b1e944e 100644 --- a/src/os_compat.c +++ b/src/os_compat.c @@ -10,6 +10,7 @@ #include <stdint.h> #endif #include "os_compat.h" +#include "os_internal.h" #include <errno.h> #include <stdio.h> @@ -26,6 +27,7 @@ #include <process.h> #include <windows.h> #else +#include <dirent.h> #include <fcntl.h> #include <poll.h> #include <pthread.h> @@ -93,26 +95,10 @@ int os_path_join(char *out, size_t out_sz, const char *a, const char *b) { return 0; } +#ifndef _WIN32 int os_realpath(const char *path, char *out, size_t out_sz) { if (!path || !out || out_sz < 2) return -1; -#ifdef _WIN32 - char buf[OS_PATH_MAX]; - DWORD n = GetFullPathNameA(path, (DWORD)sizeof(buf), buf, NULL); - if (n == 0 || n >= sizeof(buf)) - return -1; - /* Prefer forward slashes internally for sandbox compares. */ - os_path_norm_slashes(buf); - if (strlen(buf) + 1 > out_sz) - return -1; - memcpy(out, buf, strlen(buf) + 1); - /* Lowercase drive letter for stable compares */ - if (out[0] && out[1] == ':') { - if (out[0] >= 'A' && out[0] <= 'Z') - out[0] = (char)(out[0] - 'A' + 'a'); - } - return 0; -#else char *r = realpath(path, NULL); if (!r) return -1; @@ -123,7 +109,6 @@ int os_realpath(const char *path, char *out, size_t out_sz) { memcpy(out, r, strlen(r) + 1); free(r); return 0; -#endif } int os_path_under(const char *root, const char *path) { @@ -132,14 +117,9 @@ int os_path_under(const char *root, const char *path) { size_t rl = strlen(root); if (strncmp(path, root, rl) != 0) return 0; - if (path[rl] == '\0') - return 1; -#ifdef _WIN32 - return path[rl] == '/' || path[rl] == '\\'; -#else - return path[rl] == '/'; -#endif + return path[rl] == '\0' || path[rl] == '/'; } +#endif int os_mkdir(const char *path, int mode) { if (!path) @@ -172,16 +152,93 @@ int os_mkdir_p(const char *path, int mode) { return os_mkdir(tmp, mode); } +#ifndef _WIN32 +struct os_dir { + DIR *d; + char path[OS_PATH_MAX]; + os_dirent_t ent; +}; + +os_dir_t *os_dir_open(const char *path) { + if (!path) + return NULL; + os_dir_t *d = calloc(1, sizeof(*d)); + if (!d) + return NULL; + snprintf(d->path, sizeof(d->path), "%s", path); + d->d = opendir(path); + if (!d->d) { + free(d); + return NULL; + } + return d; +} + +const os_dirent_t *os_dir_next(os_dir_t *d) { + struct dirent *de; + while (d && (de = readdir(d->d)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + d->ent.name = de->d_name; + d->ent.is_dir = d->ent.is_link = 0; +#ifdef DT_DIR + if (de->d_type != DT_UNKNOWN) { + d->ent.is_dir = de->d_type == DT_DIR; + d->ent.is_link = de->d_type == DT_LNK; + return &d->ent; + } +#endif + char p[OS_PATH_MAX]; + struct stat st; + if (snprintf(p, sizeof(p), "%s/%s", d->path, de->d_name) < + (int)sizeof(p) && + lstat(p, &st) == 0) { + d->ent.is_dir = S_ISDIR(st.st_mode); + d->ent.is_link = S_ISLNK(st.st_mode); + } + return &d->ent; + } + return NULL; +} + +void os_dir_close(os_dir_t *d) { + if (!d) + return; + closedir(d->d); + free(d); +} + +int os_lstat(const char *path, os_stat_t *st) { + struct stat s; + memset(st, 0, sizeof(*st)); + if (!path || lstat(path, &s) != 0) + return -1; + st->is_dir = S_ISDIR(s.st_mode); + st->is_reg = S_ISREG(s.st_mode); + st->is_link = S_ISLNK(s.st_mode); + st->size = (long long)s.st_size; + st->mtime = (double)s.st_mtime; + return 0; +} + +int os_rename_replace(const char *from, const char *to) { + return rename(from, to) == 0 ? 0 : -1; +} +#endif + int os_config_dir(char *out, size_t out_sz) { if (!out || out_sz < 8) return -1; #ifdef _WIN32 - const char *base = getenv("APPDATA"); + const char *base = getenv("XDG_CONFIG_HOME"); + if (!base || !os_path_is_abs(base)) + base = getenv("APPDATA"); if (!base || !base[0]) base = getenv("LOCALAPPDATA"); if (!base || !base[0]) return -1; - int n = snprintf(out, out_sz, "%s\\cst-agent", base); + int n = snprintf(out, out_sz, "%s/cst-agent", base); + os_path_norm_slashes(out); #else const char *xdg = getenv("XDG_CONFIG_HOME"); const char *home = getenv("HOME"); @@ -389,7 +446,7 @@ void os_set_run_timeout(int seconds) { g_run_timeout_s = seconds; } int os_get_run_timeout(void) { return g_run_timeout_s; } -static int run_timeout_s(void) { +int os_run_timeout_effective(void) { if (g_run_timeout_s > 0) return g_run_timeout_s; const char *e = getenv("CST_SHELL_TIMEOUT"); @@ -414,8 +471,8 @@ double os_now_ms(void) { } /* Append to capture buffer, keeping at most max_out bytes. */ -static int cap_append(char **buf, size_t *len, size_t *cap, size_t max_out, - const char *data, size_t n, int *truncated) { +int os_cap_append(char **buf, size_t *len, size_t *cap, size_t max_out, + const char *data, size_t n, int *truncated) { if (*len >= max_out) { *truncated = 1; return 0; @@ -440,7 +497,7 @@ static int cap_append(char **buf, size_t *len, size_t *cap, size_t max_out, return 0; } -static void cap_note(char **buf, size_t *len, size_t *cap, const char *note) { +void os_cap_note(char **buf, size_t *len, size_t *cap, const char *note) { size_t n = strlen(note); if (*len + n + 1 > *cap) { char *nb = realloc(*buf, *len + n + 1); @@ -453,154 +510,7 @@ static void cap_note(char **buf, size_t *len, size_t *cap, const char *note) { *len += n; } -#ifdef _WIN32 -int os_run_capture(const char *cwd, const char *const argv[], size_t max_out, - os_run_result_t *r) { - if (!argv || !argv[0] || !r) - return -1; - if (max_out == 0) - max_out = 512 * 1024; - memset(r, 0, sizeof(*r)); - - char cmdline[8192]; - size_t pos = 0; - cmdline[0] = 0; - for (int i = 0; argv[i]; i++) { - const char *a = argv[i]; - int need_q = strchr(a, ' ') || strchr(a, '\t') || strchr(a, '"'); - if (i && pos + 1 < sizeof(cmdline)) - cmdline[pos++] = ' '; - if (need_q) { - if (pos + 1 < sizeof(cmdline)) - cmdline[pos++] = '"'; - for (const char *p = a; *p && pos + 2 < sizeof(cmdline); p++) { - if (*p == '"') { - cmdline[pos++] = '\\'; - cmdline[pos++] = '"'; - } else { - cmdline[pos++] = *p; - } - } - if (pos + 1 < sizeof(cmdline)) - cmdline[pos++] = '"'; - } else { - size_t L = strlen(a); - if (pos + L >= sizeof(cmdline)) - break; - memcpy(cmdline + pos, a, L); - pos += L; - } - cmdline[pos] = 0; - } - - SECURITY_ATTRIBUTES sa = {0}; - sa.nLength = sizeof(sa); - sa.bInheritHandle = TRUE; - HANDLE rd = NULL, wr = NULL; - if (!CreatePipe(&rd, &wr, &sa, 0)) - return -1; - SetHandleInformation(rd, HANDLE_FLAG_INHERIT, 0); - - STARTUPINFOA si = {0}; - si.cb = sizeof(si); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdOutput = wr; - si.hStdError = wr; - HANDLE nul = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, - &sa, OPEN_EXISTING, 0, NULL); - si.hStdInput = nul != INVALID_HANDLE_VALUE ? nul : GetStdHandle(STD_INPUT_HANDLE); - PROCESS_INFORMATION pi = {0}; - - char *cmd_mut = _strdup(cmdline); - if (!cmd_mut) { - CloseHandle(rd); - CloseHandle(wr); - return -1; - } - BOOL ok = CreateProcessA(NULL, cmd_mut, NULL, NULL, TRUE, 0, NULL, - cwd && cwd[0] ? cwd : NULL, &si, &pi); - free(cmd_mut); - CloseHandle(wr); - if (nul != INVALID_HANDLE_VALUE) - CloseHandle(nul); - if (!ok) { - CloseHandle(rd); - return -1; - } - - size_t cap = 4096, len = 0; - char *buf = malloc(cap); - if (!buf) { - CloseHandle(rd); - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - return -1; - } - buf[0] = 0; - int truncated = 0, killed = 0; - double deadline = os_now_ms() + run_timeout_s() * 1000.0; - for (;;) { - DWORD avail = 0; - if (!PeekNamedPipe(rd, NULL, 0, NULL, &avail, NULL)) - break; /* writer closed */ - if (avail > 0) { - char tmp[4096]; - DWORD n = 0; - if (!ReadFile(rd, tmp, sizeof(tmp), &n, NULL) || n == 0) - break; - cap_append(&buf, &len, &cap, max_out, tmp, n, &truncated); - continue; - } - if (WaitForSingleObject(pi.hProcess, 50) == WAIT_OBJECT_0) { - /* drain what is left */ - while (PeekNamedPipe(rd, NULL, 0, NULL, &avail, NULL) && avail > 0) { - char tmp[4096]; - DWORD n = 0; - if (!ReadFile(rd, tmp, sizeof(tmp), &n, NULL) || n == 0) - break; - cap_append(&buf, &len, &cap, max_out, tmp, n, &truncated); - } - break; - } - if ((g_run_cancel && *g_run_cancel) || os_now_ms() > deadline) { - killed = (g_run_cancel && *g_run_cancel) ? 2 : 1; - TerminateProcess(pi.hProcess, killed == 1 ? 124 : 130); - break; - } - } - CloseHandle(rd); - WaitForSingleObject(pi.hProcess, INFINITE); - DWORD code = 1; - GetExitCodeProcess(pi.hProcess, &code); - if (truncated) - cap_note(&buf, &len, &cap, "\n...(output truncated)\n"); - if (killed == 1) { - char note[96]; - snprintf(note, sizeof(note), "\n[cst: killed after %ds timeout]\n", - run_timeout_s()); - cap_note(&buf, &len, &cap, note); - code = 124; - } else if (killed == 2) { - cap_note(&buf, &len, &cap, "\n[cst: cancelled]\n"); - code = 130; - } - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - buf[len] = 0; - r->out = buf; - r->out_len = len; - r->exit_code = (int)code; - return 0; -} - -int os_run_capture_ex(const char *cwd, const char *const argv[], - size_t max_out, os_run_result_t *r, - os_child_hook_fn child_hook, void *ud) { - (void)child_hook; - (void)ud; - return os_run_capture(cwd, argv, max_out, r); -} -#else +#ifndef _WIN32 static void cloexec(int fd) { int fl = fcntl(fd, F_GETFD); if (fl >= 0) @@ -685,7 +595,7 @@ int os_run_capture_ex(const char *cwd, const char *const argv[], int truncated = 0, killed = 0; int status = 0, reaped = 0, bg_killed = 0; double reaped_at = 0, killed_at = 0; - double deadline = os_now_ms() + run_timeout_s() * 1000.0; + double deadline = os_now_ms() + os_run_timeout_effective() * 1000.0; for (;;) { struct pollfd pfd = {pipefd[0], POLLIN, 0}; int pr = poll(&pfd, 1, 100); @@ -697,7 +607,7 @@ int os_run_capture_ex(const char *cwd, const char *const argv[], if (n == 0) break; /* every writer closed */ if (n > 0) - cap_append(&buf, &len, &cap, max_out, tmp, (size_t)n, &truncated); + os_cap_append(&buf, &len, &cap, max_out, tmp, (size_t)n, &truncated); else if (errno != EINTR && errno != EAGAIN) break; } @@ -733,9 +643,9 @@ int os_run_capture_ex(const char *cwd, const char *const argv[], waitpid(pid, &status, 0); } if (truncated) - cap_note(&buf, &len, &cap, "\n...(output truncated)\n"); + os_cap_note(&buf, &len, &cap, "\n...(output truncated)\n"); if (bg_killed) - cap_note(&buf, &len, &cap, + os_cap_note(&buf, &len, &cap, "\n[cst: background processes left running were stopped]\n"); r->out = buf; r->out_len = len; @@ -743,11 +653,11 @@ int os_run_capture_ex(const char *cwd, const char *const argv[], if (killed == 1) { char note[96]; snprintf(note, sizeof(note), "\n[cst: killed after %ds timeout]\n", - run_timeout_s()); - cap_note(&r->out, &r->out_len, &cap, note); + os_run_timeout_effective()); + os_cap_note(&r->out, &r->out_len, &cap, note); r->exit_code = 124; } else if (killed == 2) { - cap_note(&r->out, &r->out_len, &cap, "\n[cst: cancelled]\n"); + os_cap_note(&r->out, &r->out_len, &cap, "\n[cst: cancelled]\n"); r->exit_code = 130; } return 0; @@ -759,40 +669,64 @@ int os_run_shell_c(const char *cwd, const char *cmd, size_t max_out, return os_run_shell_ex(cwd, cmd, max_out, r, NULL, NULL); } +/* busybox: busybox sh -c cmd ; /bin/sh: sh -c cmd. argv needs 5 slots and + * points into shell. */ +static int shell_argv(char *shell, size_t shell_sz, const char *cmd, + const char *argv[5]) { + if (!cmd || os_resolve_shell(shell, shell_sz) != 0) + return -1; + const char *base = shell; + const char *s = strrchr(shell, '/'); +#ifdef _WIN32 + const char *s2 = strrchr(shell, '\\'); + if (s2 && (!s || s2 > s)) + s = s2; +#endif + if (s) + base = s + 1; + int i = 0; + argv[i++] = shell; + if (strncmp(base, "busybox", 7) == 0) + argv[i++] = "sh"; + argv[i++] = "-c"; + argv[i++] = cmd; + argv[i] = NULL; + return 0; +} + int os_run_shell_ex(const char *cwd, const char *cmd, size_t max_out, os_run_result_t *r, os_child_hook_fn child_hook, void *ud) { - if (!cmd || !r) - return -1; char shell[OS_PATH_MAX]; - if (os_resolve_shell(shell, sizeof(shell)) != 0) + const char *argv[5]; + if (!r || shell_argv(shell, sizeof(shell), cmd, argv) != 0) return -1; - - /* busybox: busybox sh -c cmd ; /bin/sh: sh -c cmd */ - int is_busybox = 0; - { - const char *base = shell; - const char *s = strrchr(shell, '/'); -#ifdef _WIN32 - const char *s2 = strrchr(shell, '\\'); - if (s2 && (!s || s2 > s)) - s = s2; -#endif - if (s) - base = s + 1; - if (strncmp(base, "busybox", 7) == 0) - is_busybox = 1; - } - - if (is_busybox) { - const char *argv[] = {shell, "sh", "-c", cmd, NULL}; - return os_run_capture_ex(cwd, argv, max_out, r, child_hook, ud); - } else { - const char *argv[] = {shell, "-c", cmd, NULL}; - return os_run_capture_ex(cwd, argv, max_out, r, child_hook, ud); - } + return os_run_capture_ex(cwd, argv, max_out, r, child_hook, ud); } +#ifdef _WIN32 +int os_run_shell_token(const char *cwd, const char *cmd, size_t max_out, + os_run_result_t *r, void *token, + const char *const env_add[]) { + char shell[OS_PATH_MAX]; + const char *argv[5]; + if (!r || shell_argv(shell, sizeof(shell), cmd, argv) != 0) + return -1; + return os_run_capture_token(cwd, argv, max_out, r, token, env_add); +} +#endif + +#ifndef _WIN32 +int os_console_init(void) { return 0; } + +char *os_output_to_utf8(const char *s, size_t n, size_t *out_len) { + (void)s; + (void)n; + (void)out_len; + return NULL; +} +#endif + void os_localtime(const time_t *t, struct tm *out) { if (!t || !out) return; @@ -811,16 +745,13 @@ int os_getpid(void) { #endif } +#ifndef _WIN32 int os_chmod_private(const char *path) { if (!path) return -1; -#ifdef _WIN32 - (void)path; - return 0; -#else return chmod(path, 0600); -#endif } +#endif typedef struct { os_task_fn fn; @@ -905,133 +836,7 @@ int os_cancel_requested(void) { return g_run_cancel && *g_run_cancel; } volatile int *os_get_cancel_flag(void) { return g_run_cancel; } -#ifdef _WIN32 -static void win_quote_arg(char *out, size_t out_sz, size_t *pos, - const char *a) { - int need_q = !a[0] || strpbrk(a, " \t\"") != NULL; - if (need_q && *pos + 1 < out_sz) - out[(*pos)++] = '"'; - for (const char *p = a; *p && *pos + 2 < out_sz; p++) { - if (*p == '"') - out[(*pos)++] = '\\'; - out[(*pos)++] = *p; - } - if (need_q && *pos + 1 < out_sz) - out[(*pos)++] = '"'; - out[*pos] = 0; -} - -int os_proc_spawn(const char *cwd, const char *const argv[], - const char *const env_add[], const char *stderr_path, - os_proc_t *p) { - memset(p, 0, sizeof(*p)); - if (!argv || !argv[0]) - return -1; - char cmdline[16384]; - size_t pos = 0; - cmdline[0] = 0; - for (int i = 0; argv[i]; i++) { - if (i && pos + 1 < sizeof(cmdline)) - cmdline[pos++] = ' '; - win_quote_arg(cmdline, sizeof(cmdline), &pos, argv[i]); - } - for (int i = 0; env_add && env_add[i]; i++) - _putenv(env_add[i]); /* inherited by the child */ - SECURITY_ATTRIBUTES sa = {0}; - sa.nLength = sizeof(sa); - sa.bInheritHandle = TRUE; - HANDLE in_r = NULL, in_w = NULL, out_r = NULL, out_w = NULL; - if (!CreatePipe(&in_r, &in_w, &sa, 0)) - return -1; - if (!CreatePipe(&out_r, &out_w, &sa, 0)) { - CloseHandle(in_r); - CloseHandle(in_w); - return -1; - } - SetHandleInformation(in_w, HANDLE_FLAG_INHERIT, 0); - SetHandleInformation(out_r, HANDLE_FLAG_INHERIT, 0); - HANDLE err = CreateFileA(stderr_path ? stderr_path : "NUL", GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, - CREATE_ALWAYS, 0, NULL); - STARTUPINFOA si = {0}; - si.cb = sizeof(si); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = in_r; - si.hStdOutput = out_w; - si.hStdError = err != INVALID_HANDLE_VALUE ? err : out_w; - PROCESS_INFORMATION pi = {0}; - BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, TRUE, CREATE_NO_WINDOW, - NULL, cwd && cwd[0] ? cwd : NULL, &si, &pi); - CloseHandle(in_r); - CloseHandle(out_w); - if (err != INVALID_HANDLE_VALUE) - CloseHandle(err); - if (!ok) { - CloseHandle(in_w); - CloseHandle(out_r); - return -1; - } - CloseHandle(pi.hThread); - p->proc = pi.hProcess; - p->in_w = in_w; - p->out_r = out_r; - return 0; -} - -int os_proc_write(os_proc_t *p, const char *data, size_t n) { - while (n > 0) { - DWORD w = 0; - if (!p->in_w || !WriteFile((HANDLE)p->in_w, data, (DWORD)n, &w, NULL) || - w == 0) - return -1; - data += w; - n -= w; - } - return 0; -} - -long os_proc_read(os_proc_t *p, char *buf, size_t n, int timeout_ms) { - if (!p->out_r) - return -1; - double deadline = os_now_ms() + timeout_ms; - for (;;) { - DWORD avail = 0; - if (!PeekNamedPipe((HANDLE)p->out_r, NULL, 0, NULL, &avail, NULL)) - return GetLastError() == ERROR_BROKEN_PIPE ? 0 : -1; - if (avail > 0) - break; - if (os_now_ms() >= deadline) - return -2; - Sleep(10); - } - DWORD r = 0; - if (!ReadFile((HANDLE)p->out_r, buf, (DWORD)n, &r, NULL)) - return GetLastError() == ERROR_BROKEN_PIPE ? 0 : -1; - return (long)r; -} - -void os_proc_close_stdin(os_proc_t *p) { - if (p->in_w) { - CloseHandle((HANDLE)p->in_w); - p->in_w = NULL; - } -} - -void os_proc_kill(os_proc_t *p) { - os_proc_close_stdin(p); - if (p->proc) { - if (WaitForSingleObject((HANDLE)p->proc, 500) != WAIT_OBJECT_0) - TerminateProcess((HANDLE)p->proc, 1); - WaitForSingleObject((HANDLE)p->proc, 2000); - CloseHandle((HANDLE)p->proc); - p->proc = NULL; - } - if (p->out_r) { - CloseHandle((HANDLE)p->out_r); - p->out_r = NULL; - } -} -#else +#ifndef _WIN32 int os_proc_spawn(const char *cwd, const char *const argv[], const char *const env_add[], const char *stderr_path, os_proc_t *p) { diff --git a/src/os_internal.h b/src/os_internal.h new file mode 100644 index 0000000..8c8f62e --- /dev/null +++ b/src/os_internal.h @@ -0,0 +1,14 @@ +/* Shared between os_compat.c and os_win.c; not part of the public API. */ +#ifndef CST_OS_INTERNAL_H +#define CST_OS_INTERNAL_H + +#include <stddef.h> + +/* CST_SHELL_TIMEOUT / os_set_run_timeout / 120 s */ +int os_run_timeout_effective(void); +/* Append to a capture buffer, keeping at most max_out bytes. */ +int os_cap_append(char **buf, size_t *len, size_t *cap, size_t max_out, + const char *data, size_t n, int *truncated); +void os_cap_note(char **buf, size_t *len, size_t *cap, const char *note); + +#endif diff --git a/src/os_win.c b/src/os_win.c new file mode 100644 index 0000000..67f35aa --- /dev/null +++ b/src/os_win.c @@ -0,0 +1,934 @@ +/* Windows side of os_compat: every string crossing the OS boundary is UTF-8 + * inside cst-agent and UTF-16 at the Win32 W APIs. The embedded manifest also + * sets the process code page to UTF-8, so CRT calls (fopen, getenv, argv) + * see UTF-8 too. */ +#ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include "os_compat.h" +#include "cst_util.h" +#include "os_internal.h" + +#include <locale.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <windows.h> + +#include <aclapi.h> +#include <sddl.h> + +wchar_t *os_utf8_to_wide(const char *s) { + if (!s) + return NULL; + int n = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0); + if (n <= 0) + return NULL; + wchar_t *w = malloc((size_t)n * sizeof(wchar_t)); + if (w && MultiByteToWideChar(CP_UTF8, 0, s, -1, w, n) <= 0) { + free(w); + return NULL; + } + return w; +} + +char *os_wide_to_utf8(const wchar_t *w) { + if (!w) + return NULL; + int n = WideCharToMultiByte(CP_UTF8, 0, w, -1, NULL, 0, NULL, NULL); + if (n <= 0) + return NULL; + char *s = malloc((size_t)n); + if (s && WideCharToMultiByte(CP_UTF8, 0, w, -1, s, n, NULL, NULL) <= 0) { + free(s); + return NULL; + } + return s; +} + +static UINT g_old_cp, g_old_out_cp; + +static void console_restore(void) { + if (g_old_cp) + SetConsoleCP(g_old_cp); + if (g_old_out_cp) + SetConsoleOutputCP(g_old_out_cp); +} + +int os_console_init(void) { + static int done; + if (done) + return GetACP() == CP_UTF8 ? 0 : -1; + done = 1; + g_old_cp = GetConsoleCP(); + g_old_out_cp = GetConsoleOutputCP(); + if (g_old_cp || g_old_out_cp) { + SetConsoleCP(CP_UTF8); + SetConsoleOutputCP(CP_UTF8); + atexit(console_restore); + } + setlocale(LC_ALL, ".UTF-8"); + /* children inherit these: Python otherwise writes the ANSI code page to pipes */ + if (!getenv("PYTHONUTF8")) + _putenv_s("PYTHONUTF8", "1"); + if (!getenv("PYTHONIOENCODING")) + _putenv_s("PYTHONIOENCODING", "utf-8"); + return GetACP() == CP_UTF8 ? 0 : -1; +} + +/* The system locale's code page. GetOEMCP()/GetACP() say 65001 once the + * manifest makes this process UTF-8, but children without one (cmd, most + * console tools) still write the system's OEM code page, e.g. 936. */ +static UINT system_cp(LCTYPE which) { + wchar_t b[8]; + if (!GetLocaleInfoW(LOCALE_SYSTEM_DEFAULT, which, b, 8)) + return 0; + return (UINT)wcstoul(b, NULL, 10); +} + +char *os_output_to_utf8(const char *s, size_t n, size_t *out_len) { + if (!s || cst_utf8_valid(s, n)) + return NULL; + UINT cps[] = {GetConsoleOutputCP(), system_cp(LOCALE_IDEFAULTCODEPAGE), + system_cp(LOCALE_IDEFAULTANSICODEPAGE), GetOEMCP(), GetACP()}; + for (size_t i = 0; i < sizeof(cps) / sizeof(cps[0]); i++) { + UINT cp = cps[i]; + if (!cp || cp == CP_UTF8) + continue; + int wn = MultiByteToWideChar(cp, MB_ERR_INVALID_CHARS, s, (int)n, NULL, 0); + if (wn <= 0) + continue; + wchar_t *w = malloc((size_t)wn * sizeof(wchar_t)); + if (!w) + return NULL; + MultiByteToWideChar(cp, 0, s, (int)n, w, wn); + int un = WideCharToMultiByte(CP_UTF8, 0, w, wn, NULL, 0, NULL, NULL); + char *u = un > 0 ? malloc((size_t)un + 1) : NULL; + if (u) { + WideCharToMultiByte(CP_UTF8, 0, w, wn, u, un, NULL, NULL); + u[un] = 0; + if (out_len) + *out_len = (size_t)un; + } + free(w); + return u; + } + return NULL; +} + +/* --- paths --------------------------------------------------------------- */ + +/* UTF-8 path -> wide, with the \\?\ prefix once it gets near MAX_PATH so + * deep trees work without the LongPathsEnabled policy. */ +static wchar_t *wpath(const char *p) { + wchar_t *w = os_utf8_to_wide(p); + if (!w || wcslen(w) < MAX_PATH - 12) + return w; + DWORD n = GetFullPathNameW(w, 0, NULL, NULL); + wchar_t *f = n ? malloc(n * sizeof(wchar_t)) : NULL; + wchar_t *out = n ? malloc((n + 8) * sizeof(wchar_t)) : NULL; + if (!f || !out || GetFullPathNameW(w, n, f, NULL) == 0) { + free(f); + free(out); + return w; + } + free(w); + if (wcsncmp(f, L"\\\\?\\", 4) == 0) { + wcscpy(out, f); + } else if (f[0] == L'\\' && f[1] == L'\\') { /* \\srv\share -> \\?\UNC\srv\share */ + wcscpy(out, L"\\\\?\\UNC"); + wcscat(out, f + 1); + } else { + wcscpy(out, L"\\\\?\\"); + wcscat(out, f); + } + free(f); + return out; +} + +/* Wide path from the OS -> UTF-8 with '/' separators, no \\?\ prefix and a + * lowercase drive letter (stable for prefix compares). */ +static int path_out(const wchar_t *w, char *out, size_t out_sz) { + const char *lead = ""; + if (wcsncmp(w, L"\\\\?\\UNC\\", 8) == 0) { + lead = "\\\\"; /* \\?\UNC\srv\share -> \\srv\share */ + w += 8; + } else if (wcsncmp(w, L"\\\\?\\", 4) == 0) { + w += 4; + } + char *u = os_wide_to_utf8(w); + if (!u) + return -1; + int n = snprintf(out, out_sz, "%s%s", lead, u); + free(u); + if (n < 0 || (size_t)n >= out_sz) + return -1; + os_path_norm_slashes(out); + if (out[0] >= 'A' && out[0] <= 'Z' && out[1] == ':') + out[0] = (char)(out[0] - 'A' + 'a'); + return 0; +} + +int os_realpath(const char *path, char *out, size_t out_sz) { + if (!path || !path[0] || !out || out_sz < 2) + return -1; + wchar_t *w = wpath(path); + if (!w) + return -1; + HANDLE h = CreateFileW(w, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + free(w); + if (h == INVALID_HANDLE_VALUE) + return -1; + wchar_t stackbuf[MAX_PATH + 16], *buf = stackbuf; + DWORD n = GetFinalPathNameByHandleW(h, buf, MAX_PATH + 16, + FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); + if (n >= MAX_PATH + 16) { + buf = malloc((n + 1) * sizeof(wchar_t)); + n = buf ? GetFinalPathNameByHandleW(h, buf, n + 1, + FILE_NAME_NORMALIZED | VOLUME_NAME_DOS) + : 0; + } + CloseHandle(h); + int rc = n ? path_out(buf, out, out_sz) : -1; + if (buf != stackbuf) + free(buf); + return rc; +} + +/* NTFS names are case-insensitive: compare the prefix the way the OS does. */ +int os_path_under(const char *root, const char *path) { + if (!root || !path) + return 0; + wchar_t *wr = os_utf8_to_wide(root), *wp = os_utf8_to_wide(path); + int ok = 0; + if (wr && wp) { + size_t rl = wcslen(wr); + if (wcslen(wp) >= rl && + (rl == 0 || CompareStringOrdinal(wr, (int)rl, wp, (int)rl, TRUE) == + CSTR_EQUAL)) { + wchar_t c = wp[rl]; + ok = c == 0 || c == L'/' || c == L'\\' || + (rl && (wr[rl - 1] == L'/' || wr[rl - 1] == L'\\')); + } + } + free(wr); + free(wp); + return ok; +} + +static int reparse_is_link(DWORD attrs, DWORD tag) { + return (attrs & FILE_ATTRIBUTE_REPARSE_POINT) && + (tag == IO_REPARSE_TAG_SYMLINK || tag == IO_REPARSE_TAG_MOUNT_POINT); +} + +/* --- directories ------------------------------------------------------- */ + +struct os_dir { + HANDLE h; + int pending; /* fd holds an entry not yet returned */ + WIN32_FIND_DATAW fd; + char name[MAX_PATH * 4]; + os_dirent_t ent; +}; + +os_dir_t *os_dir_open(const char *path) { + if (!path || !path[0]) + return NULL; + size_t L = strlen(path); + char *pat = malloc(L + 3); + if (!pat) + return NULL; + memcpy(pat, path, L); + if (path[L - 1] != '/' && path[L - 1] != '\\') + pat[L++] = '\\'; + pat[L++] = '*'; + pat[L] = 0; + wchar_t *w = wpath(pat); + free(pat); + os_dir_t *d = w ? calloc(1, sizeof(*d)) : NULL; + if (!d) { + free(w); + return NULL; + } + d->h = FindFirstFileExW(w, FindExInfoBasic, &d->fd, FindExSearchNameMatch, + NULL, FIND_FIRST_EX_LARGE_FETCH); + free(w); + if (d->h == INVALID_HANDLE_VALUE) { + DWORD e = GetLastError(); + if (e != ERROR_FILE_NOT_FOUND) { /* an empty drive root has no entries */ + free(d); + return NULL; + } + } else { + d->pending = 1; + } + return d; +} + +const os_dirent_t *os_dir_next(os_dir_t *d) { + if (!d || d->h == INVALID_HANDLE_VALUE) + return NULL; + for (;;) { + if (!d->pending && !FindNextFileW(d->h, &d->fd)) + return NULL; + d->pending = 0; + const wchar_t *n = d->fd.cFileName; + if (wcscmp(n, L".") == 0 || wcscmp(n, L"..") == 0) + continue; + if (WideCharToMultiByte(CP_UTF8, 0, n, -1, d->name, (int)sizeof(d->name), + NULL, NULL) <= 0) + continue; + DWORD a = d->fd.dwFileAttributes; + d->ent.name = d->name; + d->ent.is_link = reparse_is_link(a, d->fd.dwReserved0); + d->ent.is_dir = !d->ent.is_link && (a & FILE_ATTRIBUTE_DIRECTORY); + return &d->ent; + } +} + +void os_dir_close(os_dir_t *d) { + if (!d) + return; + if (d->h != INVALID_HANDLE_VALUE) + FindClose(d->h); + free(d); +} + +static double filetime_secs(FILETIME ft) { + ULONGLONG t = ((ULONGLONG)ft.dwHighDateTime << 32) | ft.dwLowDateTime; + return (double)t / 1e7 - 11644473600.0; +} + +int os_lstat(const char *path, os_stat_t *st) { + memset(st, 0, sizeof(*st)); + wchar_t *w = path ? wpath(path) : NULL; + if (!w) + return -1; + WIN32_FILE_ATTRIBUTE_DATA fa; + if (!GetFileAttributesExW(w, GetFileExInfoStandard, &fa)) { + free(w); + return -1; + } + DWORD a = fa.dwFileAttributes; + if (a & FILE_ATTRIBUTE_REPARSE_POINT) { + WIN32_FIND_DATAW fd; + HANDLE h = FindFirstFileW(w, &fd); + if (h != INVALID_HANDLE_VALUE) { + st->is_link = reparse_is_link(a, fd.dwReserved0); + FindClose(h); + } + } + free(w); + st->is_dir = !st->is_link && (a & FILE_ATTRIBUTE_DIRECTORY); + st->is_reg = !st->is_link && !(a & FILE_ATTRIBUTE_DIRECTORY); + st->size = (long long)(((ULONGLONG)fa.nFileSizeHigh << 32) | fa.nFileSizeLow); + st->mtime = filetime_secs(fa.ftLastWriteTime); + return 0; +} + +int os_rename_replace(const char *from, const char *to) { + wchar_t *wf = wpath(from), *wt = wpath(to); + int ok = wf && wt && + MoveFileExW(wf, wt, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + free(wf); + free(wt); + return ok ? 0 : -1; +} + +/* --- processes ----------------------------------------------------------- */ + +/* One argument the way CommandLineToArgvW and the CRT split it again. */ +static void cmdline_arg(cst_buf_t *b, const char *a) { + if (a[0] && !strpbrk(a, " \t\n\v\"")) { + cst_buf_puts(b, a); + return; + } + cst_buf_puts(b, "\""); + for (const char *p = a;; p++) { + size_t bs = 0; + while (*p == '\\') { + bs++; + p++; + } + if (!*p) { + for (size_t i = 0; i < bs * 2; i++) + cst_buf_puts(b, "\\"); + break; + } + for (size_t i = 0; i < (*p == '"' ? bs * 2 + 1 : bs); i++) + cst_buf_puts(b, "\\"); + cst_buf_append(b, p, 1); + } + cst_buf_puts(b, "\""); +} + +static wchar_t *build_cmdline(const char *const argv[]) { + cst_buf_t b = {0}; + for (int i = 0; argv[i]; i++) { + if (i) + cst_buf_puts(&b, " "); + cmdline_arg(&b, argv[i]); + } + wchar_t *w = b.buf ? os_utf8_to_wide(b.buf) : NULL; + cst_buf_free(&b); + return w; +} + +static size_t env_name_len(const wchar_t *e) { + const wchar_t *eq = wcschr(e + 1, L'='); /* "=C:=C:\x" hides a leading '=' */ + return eq ? (size_t)(eq - e) : wcslen(e); +} + +static int env_cmp(const void *a, const void *b) { + const wchar_t *x = *(const wchar_t *const *)a, *y = *(const wchar_t *const *)b; + return CompareStringOrdinal(x, (int)env_name_len(x), y, (int)env_name_len(y), + TRUE) - + CSTR_EQUAL; +} + +/* The inherited environment with env_add ("KEY=VALUE") layered on top, as a + * sorted CREATE_UNICODE_ENVIRONMENT block; NULL = inherit unchanged. */ +static wchar_t *env_block(const char *const env_add[]) { + if (!env_add || !env_add[0]) + return NULL; + size_t nadd = 0, ncur = 0; + while (env_add[nadd]) + nadd++; + wchar_t *cur = GetEnvironmentStringsW(); + for (const wchar_t *e = cur; e && *e; e += wcslen(e) + 1) + ncur++; + wchar_t **add = calloc(nadd, sizeof(*add)); + wchar_t **v = calloc(ncur + nadd, sizeof(*v)); + wchar_t *blk = NULL; + size_t n = 0, total = 1; + if (!add || !v) + goto out; + for (size_t i = 0; i < nadd; i++) + if (!(add[i] = os_utf8_to_wide(env_add[i]))) + goto out; + for (wchar_t *e = cur; e && *e; e += wcslen(e) + 1) { + int shadowed = 0; + for (size_t i = 0; i < nadd && !shadowed; i++) + shadowed = env_cmp(&e, &add[i]) == 0; + if (!shadowed) + v[n++] = e; + } + for (size_t i = 0; i < nadd; i++) { + int dup = 0; /* a later entry for the same name wins */ + for (size_t k = i + 1; k < nadd && !dup; k++) + dup = env_cmp(&add[i], &add[k]) == 0; + if (!dup) + v[n++] = add[i]; + } + qsort(v, n, sizeof(*v), env_cmp); + for (size_t i = 0; i < n; i++) + total += wcslen(v[i]) + 1; + blk = malloc(total * sizeof(wchar_t)); + if (blk) { + wchar_t *o = blk; + for (size_t i = 0; i < n; i++) { + size_t l = wcslen(v[i]) + 1; + memcpy(o, v[i], l * sizeof(wchar_t)); + o += l; + } + *o = 0; + } +out: + for (size_t i = 0; add && i < nadd; i++) + free(add[i]); + free(add); + free(v); + if (cur) + FreeEnvironmentStringsW(cur); + return blk; +} + +static HANDLE new_job(void) { + HANDLE job = CreateJobObjectW(NULL, NULL); + if (!job) + return NULL; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION li; + memset(&li, 0, sizeof(li)); + li.BasicLimitInformation.LimitFlags = + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | + JOB_OBJECT_LIMIT_DIE_ON_UNHANDLED_EXCEPTION; + SetInformationJobObject(job, JobObjectExtendedLimitInformation, &li, + sizeof(li)); + return job; +} + +static DWORD job_active(HANDLE job) { + JOBOBJECT_BASIC_ACCOUNTING_INFORMATION ai; + if (!job || !QueryInformationJobObject(job, + JobObjectBasicAccountingInformation, + &ai, sizeof(ai), NULL)) + return 0; + return ai.ActiveProcesses; +} + +/* Children still running after the shell exited. A child started without + * our console (the sandbox) has its own conhost.exe, which leaves the job a + * moment after its client, so give the job a short while to drain. */ +static int job_lingers(HANDLE job) { + for (int i = 0; i < 50 && job_active(job) > 0; i++) + Sleep(10); + return job_active(job) > 0; +} + +/* Start argv suspended, put it in a fresh kill-on-close job, then resume. + * Only in/out/err are inherited (PROC_THREAD_ATTRIBUTE_HANDLE_LIST), so a + * concurrent spawn on another thread never leaks into this child. */ +static int spawn(const char *cwd, const char *const argv[], + const char *const env_add[], HANDLE token, HANDLE in, + HANDLE out, HANDLE err, HANDLE *proc, HANDLE *job, + DWORD *error) { + *proc = *job = NULL; + *error = ERROR_OUTOFMEMORY; + wchar_t *cmd = build_cmdline(argv); + wchar_t *wcwd = cwd && cwd[0] ? os_utf8_to_wide(cwd) : NULL; + wchar_t *env = env_block(env_add); + HANDLE inherit[3]; + int ninh = 0; + HANDLE std3[3] = {in, out, err}; + for (int i = 0; i < 3; i++) { + int seen = !std3[i] || std3[i] == INVALID_HANDLE_VALUE; + for (int k = 0; k < ninh && !seen; k++) + seen = inherit[k] == std3[i]; + if (!seen) + inherit[ninh++] = std3[i]; + } + SIZE_T asz = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &asz); + LPPROC_THREAD_ATTRIBUTE_LIST attrs = malloc(asz); + int ok = 0; + if (cmd && (!cwd || !cwd[0] || wcwd) && (!env_add || !env_add[0] || env) && + attrs && InitializeProcThreadAttributeList(attrs, 1, 0, &asz)) { + if (UpdateProcThreadAttribute(attrs, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + inherit, (SIZE_T)ninh * sizeof(HANDLE), NULL, + NULL)) { + STARTUPINFOEXW si; + memset(&si, 0, sizeof(si)); + si.StartupInfo.cb = sizeof(si); + si.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + si.StartupInfo.hStdInput = in; + si.StartupInfo.hStdOutput = out; + si.StartupInfo.hStdError = err; + si.lpAttributeList = attrs; + /* Share our console when there is one (children then see its UTF-8 + * code page); the sandbox token cannot attach to it. */ + DWORD flags = CREATE_SUSPENDED | CREATE_UNICODE_ENVIRONMENT | + CREATE_NEW_PROCESS_GROUP | EXTENDED_STARTUPINFO_PRESENT; + if (token || !GetConsoleWindow()) + flags |= CREATE_NO_WINDOW; + PROCESS_INFORMATION pi; + memset(&pi, 0, sizeof(pi)); + ok = token ? CreateProcessAsUserW(token, NULL, cmd, NULL, NULL, TRUE, + flags, env, wcwd, &si.StartupInfo, &pi) + : CreateProcessW(NULL, cmd, NULL, NULL, TRUE, flags, env, wcwd, + &si.StartupInfo, &pi); + if (ok) { + *job = new_job(); + if (*job && !AssignProcessToJobObject(*job, pi.hProcess)) { + CloseHandle(*job); + *job = NULL; + } + ResumeThread(pi.hThread); + CloseHandle(pi.hThread); + *proc = pi.hProcess; + } else { + *error = GetLastError(); + } + } + DeleteProcThreadAttributeList(attrs); + } + free(attrs); + free(cmd); + free(wcwd); + free(env); + return ok ? 0 : -1; +} + +int os_run_capture_token(const char *cwd, const char *const argv[], + size_t max_out, os_run_result_t *r, void *token, + const char *const env_add[]) { + if (!argv || !argv[0] || !r) + return -1; + if (max_out == 0) + max_out = 512 * 1024; + memset(r, 0, sizeof(*r)); + SECURITY_ATTRIBUTES sa = {sizeof(sa), NULL, TRUE}; + HANDLE rd = NULL, wr = NULL; + if (!CreatePipe(&rd, &wr, &sa, 64 * 1024)) + return -1; + SetHandleInformation(rd, HANDLE_FLAG_INHERIT, 0); + HANDLE nul = CreateFileW(L"NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, + OPEN_EXISTING, 0, NULL); + HANDLE proc, job; + DWORD err; + int rc = spawn(cwd, argv, env_add, (HANDLE)token, nul, wr, wr, &proc, &job, + &err); + CloseHandle(wr); + if (nul != INVALID_HANDLE_VALUE) + CloseHandle(nul); + if (rc != 0) { + CloseHandle(rd); + if (err != ERROR_FILE_NOT_FOUND && err != ERROR_PATH_NOT_FOUND) + return -1; + cst_buf_t b = {0}; + cst_buf_printf(&b, "%s: command not found\n", argv[0]); + r->out_len = b.len; + r->out = cst_buf_steal(&b); + r->exit_code = 127; + return 0; + } + + size_t cap = 4096, len = 0; + char *buf = malloc(cap); + if (!buf) { + TerminateJobObject(job, 1); + TerminateProcess(proc, 1); + CloseHandle(rd); + CloseHandle(proc); + if (job) + CloseHandle(job); + return -1; + } + buf[0] = 0; + int truncated = 0, killed = 0, reaped = 0, bg_killed = 0; + double reaped_at = 0, killed_at = 0; + double deadline = os_now_ms() + os_run_timeout_effective() * 1000.0; + int idle = 0; /* empty polls in a row: wait longer once output stalls */ + for (;;) { + DWORD avail = 0; + if (!PeekNamedPipe(rd, NULL, 0, NULL, &avail, NULL)) + break; /* every writer closed */ + if (avail) { + char tmp[65536]; + DWORD n = 0; + if (!ReadFile(rd, tmp, avail < sizeof(tmp) ? avail : sizeof(tmp), &n, + NULL) || + n == 0) + break; + os_cap_append(&buf, &len, &cap, max_out, tmp, n, &truncated); + idle = 0; + } else if (reaped) { + Sleep(idle++ < 50 ? 1 : 20); + } + DWORD wait = avail ? 0 : idle++ < 50 ? 1 : 20; + if (!reaped && WaitForSingleObject(proc, wait) == WAIT_OBJECT_0) { + reaped = 1; + reaped_at = os_now_ms(); + } + double now = os_now_ms(); + /* The shell exited but a background child still holds the pipe + * (`server &`): give it a moment, then end the tree. */ + if (reaped && !killed && !bg_killed && now - reaped_at > 1000) { + TerminateJobObject(job, 1); + bg_killed = 1; + killed_at = now; + } + if (!reaped && !killed && (os_cancel_requested() || now > deadline)) { + killed = os_cancel_requested() ? 2 : 1; + killed_at = now; + TerminateJobObject(job, killed == 1 ? 124 : 130); + if (!job) + TerminateProcess(proc, killed == 1 ? 124 : 130); + } + /* a writer outside the job can keep the pipe open */ + if ((killed || bg_killed) && now - killed_at > 3000) + break; + } + CloseHandle(rd); + WaitForSingleObject(proc, killed ? 5000 : INFINITE); + DWORD code = 1; + GetExitCodeProcess(proc, &code); + if (!bg_killed && !killed && job_lingers(job)) + bg_killed = 1; /* closing the job below ends them */ + CloseHandle(proc); + if (job) + CloseHandle(job); + + size_t ulen = 0; + char *u = os_output_to_utf8(buf, len, &ulen); + if (u) { + free(buf); + buf = u; + len = ulen; + cap = ulen + 1; + } + if (truncated) + os_cap_note(&buf, &len, &cap, "\n...(output truncated)\n"); + if (bg_killed && !killed) + os_cap_note(&buf, &len, &cap, + "\n[cst: background processes left running were stopped]\n"); + if (killed == 1) { + char note[96]; + snprintf(note, sizeof(note), "\n[cst: killed after %ds timeout]\n", + os_run_timeout_effective()); + os_cap_note(&buf, &len, &cap, note); + code = 124; + } else if (killed == 2) { + os_cap_note(&buf, &len, &cap, "\n[cst: cancelled]\n"); + code = 130; + } + r->out = buf; + r->out_len = len; + r->exit_code = (int)code; + return 0; +} + +int os_run_capture(const char *cwd, const char *const argv[], size_t max_out, + os_run_result_t *r) { + return os_run_capture_token(cwd, argv, max_out, r, NULL, NULL); +} + +int os_run_capture_ex(const char *cwd, const char *const argv[], + size_t max_out, os_run_result_t *r, + os_child_hook_fn child_hook, void *ud) { + (void)child_hook; + (void)ud; + return os_run_capture_token(cwd, argv, max_out, r, NULL, NULL); +} + +int os_proc_spawn(const char *cwd, const char *const argv[], + const char *const env_add[], const char *stderr_path, + os_proc_t *p) { + memset(p, 0, sizeof(*p)); + if (!argv || !argv[0]) + return -1; + SECURITY_ATTRIBUTES sa = {sizeof(sa), NULL, TRUE}; + HANDLE in_r = NULL, in_w = NULL, out_r = NULL, out_w = NULL; + if (!CreatePipe(&in_r, &in_w, &sa, 64 * 1024)) + return -1; + if (!CreatePipe(&out_r, &out_w, &sa, 64 * 1024)) { + CloseHandle(in_r); + CloseHandle(in_w); + return -1; + } + SetHandleInformation(in_w, HANDLE_FLAG_INHERIT, 0); + SetHandleInformation(out_r, HANDLE_FLAG_INHERIT, 0); + wchar_t *werr = stderr_path ? wpath(stderr_path) : NULL; + HANDLE err = CreateFileW(werr ? werr : L"NUL", GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, + werr ? CREATE_ALWAYS : OPEN_EXISTING, 0, NULL); + free(werr); + HANDLE proc, job; + DWORD e; + int rc = spawn(cwd, argv, env_add, NULL, in_r, out_w, + err != INVALID_HANDLE_VALUE ? err : out_w, &proc, &job, &e); + CloseHandle(in_r); + CloseHandle(out_w); + if (err != INVALID_HANDLE_VALUE) + CloseHandle(err); + if (rc != 0) { + CloseHandle(in_w); + CloseHandle(out_r); + return -1; + } + p->proc = proc; + p->job = job; + p->in_w = in_w; + p->out_r = out_r; + return 0; +} + +int os_proc_write(os_proc_t *p, const char *data, size_t n) { + while (n > 0) { + DWORD w = 0; + if (!p->in_w || !WriteFile((HANDLE)p->in_w, data, (DWORD)n, &w, NULL) || + w == 0) + return -1; + data += w; + n -= w; + } + return 0; +} + +long os_proc_read(os_proc_t *p, char *buf, size_t n, int timeout_ms) { + if (!p->out_r) + return -1; + double deadline = os_now_ms() + timeout_ms; + DWORD avail = 0; + for (;;) { + if (!PeekNamedPipe((HANDLE)p->out_r, NULL, 0, NULL, &avail, NULL)) + return GetLastError() == ERROR_BROKEN_PIPE ? 0 : -1; + if (avail > 0) + break; + if (os_now_ms() >= deadline) + return -2; + Sleep(5); + } + DWORD r = 0; + if (!ReadFile((HANDLE)p->out_r, buf, avail < n ? avail : (DWORD)n, &r, NULL)) + return GetLastError() == ERROR_BROKEN_PIPE ? 0 : -1; + return (long)r; +} + +void os_proc_close_stdin(os_proc_t *p) { + if (p->in_w) { + CloseHandle((HANDLE)p->in_w); + p->in_w = NULL; + } +} + +void os_proc_kill(os_proc_t *p) { + os_proc_close_stdin(p); + if (p->proc) { + /* polite first: most servers exit when stdin closes */ + if (WaitForSingleObject((HANDLE)p->proc, 500) != WAIT_OBJECT_0) { + if (p->job) + TerminateJobObject((HANDLE)p->job, 1); + else + TerminateProcess((HANDLE)p->proc, 1); + } + WaitForSingleObject((HANDLE)p->proc, 2000); + CloseHandle((HANDLE)p->proc); + p->proc = NULL; + } + if (p->job) { + CloseHandle((HANDLE)p->job); /* ends anything the server left behind */ + p->job = NULL; + } + if (p->out_r) { + CloseHandle((HANDLE)p->out_r); + p->out_r = NULL; + } +} + +void os_windows_version(char *out, size_t n) { + typedef LONG(WINAPI * rtl_ver_fn)(OSVERSIONINFOW *); + OSVERSIONINFOW vi; + memset(&vi, 0, sizeof(vi)); + vi.dwOSVersionInfoSize = sizeof(vi); + HMODULE nt = GetModuleHandleW(L"ntdll.dll"); + rtl_ver_fn fn = nt ? (rtl_ver_fn)(void (*)(void))GetProcAddress(nt, "RtlGetVersion") + : NULL; + SYSTEM_INFO si; + GetNativeSystemInfo(&si); + const char *arch = si.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_ARM64 + ? "arm64" + : "x86_64"; + if (fn && fn(&vi) == 0) + snprintf(out, n, "Windows %lu.%lu.%lu %s", vi.dwMajorVersion, + vi.dwMinorVersion, vi.dwBuildNumber, arch); + else + snprintf(out, n, "Windows %s", arch); +} + +/* Mandatory label from an SDDL SACL such as "S:(ML;;NWNR;;;ME)". */ +int os_win_set_label(const char *path, const char *sddl) { + wchar_t *wp = wpath(path), *ws = os_utf8_to_wide(sddl); + PSECURITY_DESCRIPTOR sd = NULL; + int rc = -1; + if (wp && ws && + ConvertStringSecurityDescriptorToSecurityDescriptorW(ws, SDDL_REVISION_1, + &sd, NULL)) { + PACL sacl = NULL; + BOOL present = FALSE, defaulted = FALSE; + if (GetSecurityDescriptorSacl(sd, &present, &sacl, &defaulted) && present && + SetNamedSecurityInfoW(wp, SE_FILE_OBJECT, LABEL_SECURITY_INFORMATION, + NULL, NULL, NULL, sacl) == ERROR_SUCCESS) + rc = 0; + LocalFree(sd); + } + free(wp); + free(ws); + return rc; +} + +int os_win_get_label(const char *path, char *out, size_t n) { + wchar_t *wp = wpath(path); + PSECURITY_DESCRIPTOR sd = NULL; + wchar_t *ws = NULL; + int rc = -1; + if (wp && n > 2 && + GetNamedSecurityInfoW(wp, SE_FILE_OBJECT, LABEL_SECURITY_INFORMATION, + NULL, NULL, NULL, NULL, &sd) == ERROR_SUCCESS && + ConvertSecurityDescriptorToStringSecurityDescriptorW( + sd, SDDL_REVISION_1, LABEL_SECURITY_INFORMATION, &ws, NULL)) { + char *s = os_wide_to_utf8(ws); + if (s) { + /* keep "S:" and the explicit ACEs; inherited ones carry the ID flag */ + size_t o = 0; + out[o++] = 'S'; + out[o++] = ':'; + rc = 0; + for (const char *p = strchr(s, '('); p; p = strchr(p + 1, '(')) { + const char *e = strchr(p, ')'); + if (!e) + break; + /* (type;flags;rights;...): flags are two-letter codes */ + const char *f = strchr(p, ';'); + int inherited = 0; + for (f = f && f < e ? f + 1 : e; f + 1 < e && *f != ';'; f += 2) + inherited |= f[0] == 'I' && f[1] == 'D'; + size_t len = (size_t)(e - p) + 1; + if (inherited) + continue; + if (o + len >= n) { + rc = -1; + break; + } + memcpy(out + o, p, len); + o += len; + } + out[o] = 0; + free(s); + } + } + if (ws) + LocalFree(ws); + if (sd) + LocalFree(sd); + free(wp); + return rc; +} + +int os_chmod_private(const char *path) { + if (!path) + return -1; + return os_win_set_label(path, "S:(ML;;NWNRNX;;;ME)"); +} + +/* --- clipboard ----------------------------------------------------------- */ + +int os_clipboard_set(const char *utf8, size_t n) { + cst_buf_t b = {0}; /* CRLF is what Windows text consumers expect */ + for (size_t i = 0; i < n; i++) { + if (utf8[i] == '\n' && (i == 0 || utf8[i - 1] != '\r')) + cst_buf_puts(&b, "\r"); + cst_buf_append(&b, utf8 + i, 1); + } + if (!b.buf) + cst_buf_puts(&b, ""); + int wn = MultiByteToWideChar(CP_UTF8, 0, b.buf, (int)b.len, NULL, 0); + HGLOBAL h = GlobalAlloc(GMEM_MOVEABLE, ((size_t)wn + 1) * sizeof(wchar_t)); + wchar_t *w = h ? GlobalLock(h) : NULL; + if (!w) { + if (h) + GlobalFree(h); + cst_buf_free(&b); + return -1; + } + MultiByteToWideChar(CP_UTF8, 0, b.buf, (int)b.len, w, wn); + w[wn] = 0; + GlobalUnlock(h); + cst_buf_free(&b); + if (!OpenClipboard(NULL)) { + GlobalFree(h); + return -1; + } + EmptyClipboard(); + int ok = SetClipboardData(CF_UNICODETEXT, h) != NULL; + if (!ok) + GlobalFree(h); + CloseClipboard(); + return ok ? 0 : -1; +} + +#endif /* _WIN32 */ diff --git a/src/permissions.c b/src/permissions.c index 56acf03..e1f687d 100644 --- a/src/permissions.c +++ b/src/permissions.c @@ -7,6 +7,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <strings.h> #include "cJSON.h" #include "cst_perms.h" @@ -33,7 +34,7 @@ static void load(cst_perms_t *p) { while ((n = fread(tmp, 1, sizeof(tmp), fp)) > 0 && b.len < 256 * 1024) cst_buf_append(&b, tmp, n); fclose(fp); - cJSON *root = b.buf ? cJSON_Parse(b.buf) : NULL; + cJSON *root = b.buf ? cJSON_Parse(b.buf + cst_bom_len(b.buf, b.len)) : NULL; cst_buf_free(&b); const cJSON *it = NULL; cJSON_ArrayForEach(it, cJSON_GetObjectItemCaseSensitive(root, "tools")) { @@ -222,6 +223,73 @@ void cst_perms_suggest_prefix(const char *cmd, char *out, size_t out_sz) { out[0] = 0; } +/* --- network commands ------------------------------------------------------ */ + +static int net_word_eq(const char *w, size_t n, const char *name) { + size_t k = strlen(name); + if (n == k + 4 && strncasecmp(w + k, ".exe", 4) == 0) + n = k; + return n == k && strncasecmp(w, name, k) == 0; +} + +int cst_shell_uses_network(const char *cmd) { + static const char *const TOOLS[] = { + "curl", "wget", "ssh", "scp", "sftp", "rsync", + "ftp", "tftp", "telnet", "nc", "ncat", "netcat", + "socat", "npx", "invoke-webrequest", "iwr", "invoke-restmethod", + "irm", "start-bitstransfer", "bitsadmin", "certutil"}; + /* tool + sub-command that downloads or uploads */ + static const char *const PAIRS[][2] = { + {"git", "clone"}, {"git", "fetch"}, {"git", "pull"}, + {"git", "push"}, {"git", "ls-remote"}, {"git", "submodule"}, + {"npm", "install"}, {"npm", "i"}, {"npm", "ci"}, + {"npm", "update"}, {"npm", "publish"}, {"npm", "exec"}, + {"pnpm", "install"}, {"pnpm", "add"}, {"pnpm", "dlx"}, + {"yarn", "install"}, {"yarn", "add"}, {"bun", "install"}, + {"bun", "add"}, {"pip", "install"}, {"pip", "download"}, + {"pip3", "install"}, {"uv", "pip"}, {"uv", "add"}, + {"uv", "sync"}, {"cargo", "install"}, {"cargo", "fetch"}, + {"cargo", "update"}, {"cargo", "publish"}, {"go", "get"}, + {"go", "install"}, {"go", "mod"}, {"gem", "install"}, + {"dotnet", "restore"}, {"dotnet", "add"}, {"winget", "install"}, + {"choco", "install"}, {"scoop", "install"}, {"apt", "install"}, + {"apt-get", "install"}, {"pkg", "install"}, {"brew", "install"}}; + if (!cmd) + return 0; + const char *prev = NULL; + size_t prev_n = 0; + for (const char *p = cmd; *p;) { + while (*p && strchr(" \t\r\n;|&()<>`'\"$=", *p)) + p++; + const char *w = p; + while (*p && !strchr(" \t\r\n;|&()<>`'\"$=", *p)) + p++; + size_t n = (size_t)(p - w); + if (!n) + break; + /* /usr/bin/curl, C:\tools\curl.exe */ + for (size_t i = n; i > 0; i--) { + if (w[i - 1] == '/' || w[i - 1] == '\\') { + n -= i; + w += i; + break; + } + } + for (size_t i = 0; i < sizeof(TOOLS) / sizeof(TOOLS[0]); i++) + if (net_word_eq(w, n, TOOLS[i])) + return 1; + for (size_t i = 0; prev && i < sizeof(PAIRS) / sizeof(PAIRS[0]); i++) + if (net_word_eq(prev, prev_n, PAIRS[i][0]) && + net_word_eq(w, n, PAIRS[i][1])) + return 1; + if (n && w[0] != '-') { + prev = w; + prev_n = n; + } + } + return 0; +} + /* --- read-only command rules ---------------------------------------------- */ #define SAFE_MAX_WORDS 64 diff --git a/src/plugins.c b/src/plugins.c index f35a491..c999f0f 100644 --- a/src/plugins.c +++ b/src/plugins.c @@ -11,10 +11,7 @@ #include <stdlib.h> #include <string.h> #include <sys/stat.h> -#ifndef _WIN32 -#include <dirent.h> #include <unistd.h> -#endif #include "cJSON.h" #include "cst_mcp.h" @@ -60,6 +57,7 @@ static char *slurp(const char *path, size_t max) { fclose(fp); if (!b.buf) cst_buf_puts(&b, ""); + cst_strip_bom(b.buf, &b.len); return cst_buf_steal(&b); } @@ -101,14 +99,6 @@ static int plugin_run(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, cst_tool_result_t *r, void *ud) { const ptool_t *pt = ud; r->exit_code = 1; -#ifdef _WIN32 - (void)ctx; - (void)c; - (void)pt; - r->out = strdup("error: plugin tools are not supported on Windows yet"); - r->out_len = r->out ? strlen(r->out) : 0; - return 0; -#else const cst_tool_def_t *def = cst_tool_find(pt->tool); char *args = cst_tool_args_json(def, c); char tmp[OS_PATH_MAX]; @@ -180,7 +170,6 @@ static int plugin_run(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, r->out_len = out.len; r->out = cst_buf_steal(&out); return 0; -#endif } /* argv array -> quoted command line; relative entries found in the plugin @@ -339,21 +328,20 @@ static void load_plugin(const char *dir, const char *dirname, static int scan(const char *root, const char *source) { int n = 0; -#ifndef _WIN32 - DIR *d = opendir(root); + os_dir_t *d = os_dir_open(root); if (!d) return 0; char *names[CST_PLUGINS_MAX * 2]; int nn = 0; - struct dirent *de; - while (nn < CST_PLUGINS_MAX * 2 && (de = readdir(d)) != NULL) { - if (de->d_name[0] == '.') + const os_dirent_t *de; + while (nn < CST_PLUGINS_MAX * 2 && (de = os_dir_next(d)) != NULL) { + if (de->name[0] == '.') continue; - names[nn] = strdup(de->d_name); + names[nn] = strdup(de->name); if (names[nn]) nn++; } - closedir(d); + os_dir_close(d); for (int i = 1; i < nn; i++) for (int j = i; j > 0 && strcmp(names[j - 1], names[j]) > 0; j--) { char *t = names[j]; @@ -371,32 +359,24 @@ static int scan(const char *root, const char *source) { } free(names[i]); } -#else - (void)root; - (void)source; -#endif return n; } static int count_plugin_dirs(const char *root) { int n = 0; -#ifndef _WIN32 - DIR *d = opendir(root); + os_dir_t *d = os_dir_open(root); if (!d) return 0; - struct dirent *de; - while ((de = readdir(d)) != NULL) { + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { char pj[OS_PATH_MAX]; - if (de->d_name[0] != '.' && - snprintf(pj, sizeof(pj), "%s/%s/plugin.json", root, de->d_name) < + if (de->name[0] != '.' && + snprintf(pj, sizeof(pj), "%s/%s/plugin.json", root, de->name) < (int)sizeof(pj) && is_file(pj)) n++; } - closedir(d); -#else - (void)root; -#endif + os_dir_close(d); return n; } diff --git a/src/sandbox_exec.c b/src/sandbox_exec.c index 9a3d750..16b04e7 100644 --- a/src/sandbox_exec.c +++ b/src/sandbox_exec.c @@ -19,6 +19,9 @@ #elif defined(__APPLE__) #define CST_HAVE_SEATBELT 1 #include <stdint.h> +#elif defined(_WIN32) +#include <windows.h> +#include <sddl.h> #endif #if defined(__linux__) && (defined(__x86_64__) || defined(__aarch64__)) @@ -864,6 +867,341 @@ static int sc_probe(void) { #endif /* CST_HAVE_SECCOMP */ +/* --- Windows: Low integrity level ----------------------------------------- */ +#ifdef _WIN32 + +#define LW_LABEL "S:(ML;OICI;NW;;;LW)" +#define LW_RECORD "sandbox-labels" + +/* Same user, no admin privileges, Low IL. No LUA_TOKEN: a deny-only + * Administrators group cannot open an admin's (SSH) window station and the + * child fails in DLL init; the integrity check does not need it. */ +static HANDLE lw_token(void) { + HANDLE self = NULL, tok = NULL; + if (!OpenProcessToken(GetCurrentProcess(), + TOKEN_DUPLICATE | TOKEN_QUERY | TOKEN_ADJUST_DEFAULT | + TOKEN_ASSIGN_PRIMARY, + &self)) + return NULL; + BOOL ok = CreateRestrictedToken(self, DISABLE_MAX_PRIVILEGE, 0, NULL, 0, + NULL, 0, NULL, &tok); + CloseHandle(self); + if (!ok) + return NULL; + PSID low = NULL; + if (!ConvertStringSidToSidW(L"S-1-16-4096", &low)) { + CloseHandle(tok); + return NULL; + } + TOKEN_MANDATORY_LABEL tml; + memset(&tml, 0, sizeof(tml)); + tml.Label.Sid = low; + tml.Label.Attributes = SE_GROUP_INTEGRITY; + ok = SetTokenInformation(tok, TokenIntegrityLevel, &tml, + (DWORD)(sizeof(tml) + GetLengthSid(low))); + LocalFree(low); + if (!ok) { + CloseHandle(tok); + return NULL; + } + return tok; +} + +static int lw_record_path(char *out, size_t n) { + char dir[OS_PATH_MAX]; + if (os_config_dir(dir, sizeof(dir)) != 0) + return -1; + return snprintf(out, n, "%s/%s", dir, LW_RECORD) < (int)n ? 0 : -1; +} + +static char *lw_read_record(void) { + char path[OS_PATH_MAX]; + if (lw_record_path(path, sizeof(path)) != 0) + return NULL; + FILE *f = fopen(path, "rb"); + if (!f) + return NULL; + size_t cap = 4096, len = 0; + char *s = malloc(cap); + size_t k; + while (s && (k = fread(s + len, 1, cap - len - 1, f)) > 0) { + len += k; + if (cap - len < 2) { + char *g = realloc(s, cap * 2); + if (!g) { + free(s); + s = NULL; + break; + } + s = g; + cap *= 2; + } + } + fclose(f); + if (s) + s[len] = 0; + return s; +} + +/* dir is a recorded directory or inside one (the label is inherited). */ +static int lw_recorded(const char *dir) { + char *s = lw_read_record(); + int found = 0; + for (char *line = s; line && *line && !found;) { + char *nl = strchr(line, '\n'); + if (nl) + *nl = 0; + char *tab = strchr(line, '\t'); + if (tab) + *tab = 0; + found = line[0] && os_path_under(line, dir); + line = nl ? nl + 1 : NULL; + } + free(s); + return found; +} + +static int lw_record_add(const char *dir, const char *old_sddl) { + char path[OS_PATH_MAX]; + if (lw_record_path(path, sizeof(path)) != 0) + return -1; + FILE *f = fopen(path, "ab"); + if (!f) + return -1; + fprintf(f, "%s\t%s\n", dir, old_sddl); + return fclose(f) == 0 ? 0 : -1; +} + +static int lw_same_dir(const char *a, const char *b) { + return a && b && a[0] && b[0] && os_path_under(a, b) && os_path_under(b, a); +} + +/* An environment directory in the form os_realpath gives the workspace + * (long names, '/'), or NULL. */ +static const char *lw_env_dir(const char *name, char *buf, size_t n) { + const char *v = getenv(name); + if (!v || !v[0] || os_realpath(v, buf, n) != 0) + return NULL; + os_path_norm_slashes(buf); + return buf; +} + +/* Labeling a whole drive, the profile or system folders would open far more + * than a project to every sandboxed command. */ +static int lw_refused(const char *dir) { + size_t n = strlen(dir); + while (n > 0 && (dir[n - 1] == '/' || dir[n - 1] == '\\')) + n--; + if (n <= 2) + return 1; + static const char *const SELF[] = {"USERPROFILE", "APPDATA", + "LOCALAPPDATA", "TEMP"}; + static const char *const TREE[] = {"SystemRoot", "ProgramFiles", + "ProgramFiles(x86)", "ProgramData"}; + char buf[OS_PATH_MAX]; + for (size_t i = 0; i < sizeof(SELF) / sizeof(SELF[0]); i++) + if (lw_same_dir(lw_env_dir(SELF[i], buf, sizeof(buf)), dir)) + return 1; + for (size_t i = 0; i < sizeof(TREE) / sizeof(TREE[0]); i++) { + const char *v = lw_env_dir(TREE[i], buf, sizeof(buf)); + if (v && os_path_under(v, dir)) + return 1; + } + return 0; +} + +static int lw_prepare(const char *ws, cst_sandbox_plan_t *p) { + static int creds_done, tmp_done; + char buf[OS_PATH_MAX + 256]; + if (!ws || !ws[0]) + return -1; + if (!creds_done) { + /* files saved by older versions have no no-read-up label yet */ + char dir[OS_PATH_MAX]; + if (os_config_dir(dir, sizeof(dir)) == 0 && + snprintf(buf, sizeof(buf), "%s/credentials", dir) < (int)sizeof(buf)) { + os_stat_t st; + if (os_lstat(buf, &st) != 0 || os_chmod_private(buf) == 0) + creds_done = 1; + } + } + char tmp[OS_PATH_MAX]; + if (os_temp_dir(tmp, sizeof(tmp)) != 0 || + snprintf(buf, sizeof(buf), "%s/cst-agent-sandbox", tmp) >= + (int)sizeof(tmp)) + return -1; + memcpy(tmp, buf, strlen(buf) + 1); + if (!tmp_done) { + os_mkdir_p(tmp, 0700); + if (os_win_set_label(tmp, LW_LABEL) != 0) + return -1; + tmp_done = 1; + } + if (!lw_recorded(ws)) { + char old[512]; + if (lw_refused(ws) || os_win_get_label(ws, old, sizeof(old)) != 0 || + os_win_set_label(ws, LW_LABEL) != 0) + return -1; + lw_record_add(ws, old); + snprintf(buf, sizeof(buf), + "[cst: sandbox: labeled %s Low integrity so sandboxed commands " + "can write there; /sandbox reset-labels undoes it]\n", + ws); + p->note = strdup(buf); + } + HANDLE tok = lw_token(); + if (!tok) { + free(p->note); + p->note = NULL; + return -1; + } + p->token = tok; + for (char *c = tmp; *c; c++) + if (*c == '/') + *c = '\\'; + p->tmpdir = strdup(tmp); + return p->tmpdir ? 0 : -1; +} + +static int lw_run(const char *ws, const char *cmd, size_t max_out, + const cst_sandbox_plan_t *p, os_run_result_t *r) { + size_t tl = strlen(p->tmpdir) + 8; + char *te = malloc(tl), *tm = malloc(tl); + int rc = -1; + if (te && tm) { + snprintf(te, tl, "TEMP=%s", p->tmpdir); + snprintf(tm, tl, "TMP=%s", p->tmpdir); + const char *env[] = {te, tm, NULL}; + rc = os_run_shell_token(ws, cmd, max_out, r, p->token, env); + } + free(te); + free(tm); + if (rc == 0 && p->note) { + size_t nl = strlen(p->note); + char *o = malloc(nl + r->out_len + 1); + if (o) { + memcpy(o, p->note, nl); + if (r->out) + memcpy(o + nl, r->out, r->out_len); + o[nl + r->out_len] = 0; + free(r->out); + r->out = o; + r->out_len += nl; + } + } + return rc; +} + +#endif /* _WIN32 */ + +char *cst_sandbox_labels(void) { +#ifdef _WIN32 + char *s = lw_read_record(); + if (!s) + return NULL; + size_t o = 0; + for (char *line = s; *line;) { + char *nl = strchr(line, '\n'); + size_t len = nl ? (size_t)(nl - line) : strlen(line); + size_t path = strcspn(line, "\t\n"); + if (path) { + memmove(s + o, line, path); + o += path; + s[o++] = '\n'; + } + line += len + (nl ? 1 : 0); + } + s[o] = 0; + if (!o) { + free(s); + return NULL; + } + return s; +#else + return NULL; +#endif +} + +int cst_sandbox_reset_labels(char **report) { + if (report) + *report = NULL; +#ifdef _WIN32 + char *s = lw_read_record(); + size_t cap = 256, len = 0; + char *rep = malloc(cap), *keep = NULL; + size_t kcap = 0, klen = 0; + int ok = 0, bad = 0; + if (rep) + rep[0] = 0; + for (char *line = s; line && *line;) { + char *nl = strchr(line, '\n'); + if (nl) + *nl = 0; + char *tab = strchr(line, '\t'); + const char *old = tab ? tab + 1 : "S:"; + if (tab) + *tab = 0; + if (line[0]) { + os_stat_t st; + int gone = os_lstat(line, &st) != 0; + int done = gone || os_win_set_label(line, old[0] ? old : "S:") == 0; + char msg[OS_PATH_MAX + 64]; + int m = snprintf(msg, sizeof(msg), "%s %s\n", + gone ? "gone " : done ? "restored" : "FAILED ", + line); + if (m > 0 && rep) { + while (len + (size_t)m + 1 > cap) { + char *g = realloc(rep, cap *= 2); + if (!g) + break; + rep = g; + } + if (len + (size_t)m + 1 <= cap) { + memcpy(rep + len, msg, (size_t)m + 1); + len += (size_t)m; + } + } + if (done) { + ok++; + } else { + bad++; + size_t need = strlen(line) + strlen(old) + 3; + if (klen + need + 1 > kcap) { + char *g = realloc(keep, kcap = (klen + need + 1) * 2); + if (g) + keep = g; + } + if (keep && klen + need + 1 <= kcap) + klen += (size_t)snprintf(keep + klen, kcap - klen, "%s\t%s\n", + line, old); + } + } + line = nl ? nl + 1 : NULL; + } + free(s); + char path[OS_PATH_MAX]; + if (lw_record_path(path, sizeof(path)) == 0) { + if (klen) { + FILE *f = fopen(path, "wb"); + if (f) { + fwrite(keep, 1, klen, f); + fclose(f); + } + } else { + remove(path); + } + } + free(keep); + if (report) + *report = rep; + else + free(rep); + return bad ? -1 : ok; +#else + return 0; +#endif +} + /* --- common --------------------------------------------------------------- */ int cst_sandbox_probe(void) { @@ -891,6 +1229,15 @@ int cst_sandbox_probe(void) { #endif #ifdef CST_HAVE_SEATBELT kind = CST_SANDBOX_SEATBELT; +#endif +#ifdef _WIN32 + if (!e || strcmp(e, "lowil") == 0) { + HANDLE tok = lw_token(); + if (tok) { + CloseHandle(tok); + kind = CST_SANDBOX_LOWIL; + } + } #endif (void)want_ll; (void)want_sc; @@ -917,6 +1264,8 @@ const char *cst_sandbox_name(int kind) { return "seatbelt"; case CST_SANDBOX_SECCOMP: return "seccomp"; + case CST_SANDBOX_LOWIL: + return "lowil"; default: return "none"; } @@ -939,6 +1288,10 @@ int cst_sandbox_prepare(const char *workspace, const char *tmpdir, #ifdef CST_HAVE_SECCOMP if (kind == CST_SANDBOX_SECCOMP) rc = sc_prepare(workspace, tmpdir, plan); +#endif +#ifdef _WIN32 + if (kind == CST_SANDBOX_LOWIL && (rc = lw_prepare(workspace, plan)) != 0) + cst_sandbox_release(plan); #endif (void)workspace; (void)tmpdir; @@ -988,6 +1341,15 @@ void cst_sandbox_release(cst_sandbox_plan_t *plan) { #ifdef CST_HAVE_SECCOMP sc_free(plan->sc); #endif +#ifdef _WIN32 + if (plan->token) + CloseHandle(plan->token); +#endif + plan->token = NULL; + free(plan->tmpdir); + plan->tmpdir = NULL; + free(plan->note); + plan->note = NULL; plan->sc = NULL; free(plan->profile); plan->profile = NULL; @@ -1004,6 +1366,12 @@ int cst_sandbox_run_shell(const char *workspace, const char *cmd, if (cst_sandbox_prepare(workspace, tmp, &plan) != 0) { memset(r, 0, sizeof(*r)); const char *msg = "[cst: sandbox unavailable; command not run]\n"; +#ifdef _WIN32 + if (workspace && !lw_recorded(workspace) && lw_refused(workspace)) + msg = "[cst: sandbox: will not label a drive root, the profile or a " + "system folder writable; command not run. Start cst-agent in a " + "project folder]\n"; +#endif r->out = malloc(strlen(msg) + 1); if (r->out) { memcpy(r->out, msg, strlen(msg) + 1); @@ -1012,8 +1380,12 @@ int cst_sandbox_run_shell(const char *workspace, const char *cmd, r->exit_code = 126; return 0; } +#ifdef _WIN32 + int rc = lw_run(workspace, cmd, max_out, &plan, r); +#else int rc = os_run_shell_ex(workspace, cmd, max_out, r, cst_sandbox_enter, &plan); +#endif cst_sandbox_release(&plan); return rc; } @@ -1039,6 +1411,11 @@ int cst_sandbox_blocked(const char *out, size_t n, int exit_code) { "权限不够", "不允许的操作", "只读文件系统", "權限不足", "拒絕不符權限的操作", "許可がありません", "Keine Berechtigung", "Permission non accord", "Permiso denegado", "Отказано в доступе", + /* Windows ERROR_ACCESS_DENIED from cmd, PowerShell and Win32 tools */ + "Access is denied", "Access to the path", "UnauthorizedAccess", + "拒绝访问", "存取被拒", "Zugriff verweigert", "Accès refusé", + /* Cygwin/MSYS2 programs cannot initialize at Low IL */ + "DLL initialization routine failed", "0xc0000142", }; for (size_t i = 0; i < sizeof(SIGNS) / sizeof(SIGNS[0]); i++) if (has(out, n, SIGNS[i])) diff --git a/src/session.c b/src/session.c index 041bb8c..39491c1 100644 --- a/src/session.c +++ b/src/session.c @@ -137,7 +137,7 @@ static const char *jstr(const cJSON *o, const char *key) { } static void load_existing(cst_session_t *s) { - FILE *rp = fopen(s->path, "r"); + FILE *rp = fopen(s->path, "rb"); if (!rp) return; char *line = NULL; @@ -196,7 +196,7 @@ cst_session_t *cst_session_open(const char *path) { pthread_mutex_init(&s->mu, NULL); snprintf(s->path, sizeof(s->path), "%s", path ? path : "session.jsonl"); load_existing(s); - s->fp = fopen(s->path, "a+"); + s->fp = fopen(s->path, "a+b"); if (!s->fp || session_grow(s) != 0) { if (s->fp) fclose(s->fp); @@ -295,7 +295,7 @@ static int reopen_truncate(cst_session_t *s) { fclose(s->fp); s->fp = NULL; } - s->fp = fopen(s->path, "w+"); + s->fp = fopen(s->path, "w+b"); return s->fp ? 0 : -1; } diff --git a/src/settings.c b/src/settings.c index 2f5aada..c3ae64c 100644 --- a/src/settings.c +++ b/src/settings.c @@ -1,4 +1,5 @@ #include "cst_agent.h" +#include "cst_util.h" #include "os_compat.h" #include "cJSON.h" @@ -31,7 +32,7 @@ int cst_settings_load(cst_settings_t *s) { size_t n = fread(buf, 1, sizeof(buf) - 1, fp); fclose(fp); buf[n] = 0; - cJSON *root = cJSON_Parse(buf); + cJSON *root = cJSON_Parse(buf + cst_bom_len(buf, n)); if (!cJSON_IsObject(root)) { cJSON_Delete(root); return -1; diff --git a/src/skills.c b/src/skills.c index b6438d1..d246aff 100644 --- a/src/skills.c +++ b/src/skills.c @@ -13,10 +13,7 @@ #include <string.h> #include <sys/stat.h> -#ifndef _WIN32 -#include <dirent.h> #include <pthread.h> -#endif #include "cst_tools.h" #include "cst_util.h" @@ -166,14 +163,9 @@ static struct { char source[48]; } g_roots[EXTRA_ROOTS]; static int g_nroots; -#ifndef _WIN32 static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; #define LOCK() pthread_mutex_lock(&g_mu) #define UNLOCK() pthread_mutex_unlock(&g_mu) -#else -#define LOCK() ((void)0) -#define UNLOCK() ((void)0) -#endif void cst_skills_add_root(const char *dir, const char *source) { if (!dir || !dir[0]) @@ -225,6 +217,7 @@ static char *slurp(const char *path, size_t max, size_t *len) { } if (!b.buf) cst_buf_puts(&b, ""); + cst_strip_bom(b.buf, &b.len); if (len) *len = b.len; return cst_buf_steal(&b); @@ -243,16 +236,15 @@ static int have(const cst_skill_t *v, int n, const char *name) { static int scan_root(const char *root, const char *source, cst_skill_t *out, int n, int max) { -#ifndef _WIN32 - DIR *d = opendir(root); + os_dir_t *d = os_dir_open(root); if (!d) return n; - struct dirent *de; - while (n < max && (de = readdir(d)) != NULL) { - if (de->d_name[0] == '.') + const os_dirent_t *de; + while (n < max && (de = os_dir_next(d)) != NULL) { + if (de->name[0] == '.') continue; char dir[OS_PATH_MAX], md[OS_PATH_MAX + 16]; - if (snprintf(dir, sizeof(dir), "%s/%s", root, de->d_name) >= + if (snprintf(dir, sizeof(dir), "%s/%s", root, de->name) >= (int)sizeof(dir)) continue; snprintf(md, sizeof(md), "%s/SKILL.md", dir); @@ -262,7 +254,7 @@ static int scan_root(const char *root, const char *source, cst_skill_t *out, cst_skill_t s; memset(&s, 0, sizeof(s)); if (!cst_fm_get(text, "name", s.name, sizeof(s.name)) || !s.name[0]) - snprintf(s.name, sizeof(s.name), "%.63s", de->d_name); + snprintf(s.name, sizeof(s.name), "%.63s", de->name); cst_fm_get(text, "description", s.desc, sizeof(s.desc)); if (!s.desc[0]) { /* first prose line of the body */ @@ -282,13 +274,7 @@ static int scan_root(const char *root, const char *source, cst_skill_t *out, if (!have(out, n, s.name)) out[n++] = s; } - closedir(d); -#else - (void)root; - (void)source; - (void)out; - (void)max; -#endif + os_dir_close(d); return n; } @@ -371,25 +357,24 @@ static int find_skill(const char *ws, const char *name, cst_skill_t *out) { static void list_files(const char *dir, const char *rel, int depth, cst_buf_t *b, int *count) { -#ifndef _WIN32 char full[OS_PATH_MAX]; if (snprintf(full, sizeof(full), "%s%s%s", dir, rel[0] ? "/" : "", rel) >= (int)sizeof(full)) return; - DIR *d = opendir(full); + os_dir_t *d = os_dir_open(full); if (!d) return; char *names[256]; int nn = 0; - struct dirent *de; - while (nn < 256 && (de = readdir(d)) != NULL) { - if (de->d_name[0] == '.') + const os_dirent_t *de; + while (nn < 256 && (de = os_dir_next(d)) != NULL) { + if (de->name[0] == '.') continue; - names[nn] = strdup(de->d_name); + names[nn] = strdup(de->name); if (names[nn]) nn++; } - closedir(d); + os_dir_close(d); for (int i = 1; i < nn; i++) { for (int j = i; j > 0 && strcmp(names[j - 1], names[j]) > 0; j--) { char *tmp = names[j]; @@ -416,13 +401,6 @@ static void list_files(const char *dir, const char *rel, int depth, } for (int i = 0; i < nn; i++) free(names[i]); -#else - (void)dir; - (void)rel; - (void)depth; - (void)b; - (void)count; -#endif } static char *errf(int *code, const char *fmt, const char *a, const char *b) { diff --git a/src/subagent.c b/src/subagent.c index 8fd2e57..32c489e 100644 --- a/src/subagent.c +++ b/src/subagent.c @@ -14,9 +14,6 @@ #include <string.h> #include <sys/stat.h> #include <time.h> -#ifndef _WIN32 -#include <dirent.h> -#endif #include "cJSON.h" #include "cst_agent.h" @@ -79,6 +76,7 @@ static char *slurp(const char *path, size_t max) { fclose(fp); if (!b.buf) cst_buf_puts(&b, ""); + cst_strip_bom(b.buf, &b.len); return cst_buf_steal(&b); } @@ -118,25 +116,24 @@ static int truthy(const char *v) { static void load_dir(const char *dir, const char *source, cst_agent_role_t *v, int *n, int max) { -#ifndef _WIN32 - DIR *d = opendir(dir); + os_dir_t *d = os_dir_open(dir); if (!d) return; - struct dirent *de; - while ((de = readdir(d)) != NULL && *n < max) { - size_t len = strlen(de->d_name); - if (de->d_name[0] == '.' || len < 4 || - strcmp(de->d_name + len - 3, ".md") != 0) + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL && *n < max) { + size_t len = strlen(de->name); + if (de->name[0] == '.' || len < 4 || + strcmp(de->name + len - 3, ".md") != 0) continue; char p[OS_PATH_MAX]; - if (snprintf(p, sizeof(p), "%s/%s", dir, de->d_name) >= (int)sizeof(p)) + if (snprintf(p, sizeof(p), "%s/%s", dir, de->name) >= (int)sizeof(p)) continue; char *text = slurp(p, ROLE_FILE_MAX); if (!text) continue; char name[64], val[1024]; if (!cst_fm_get(text, "name", name, sizeof(name)) || !name[0]) - snprintf(name, sizeof(name), "%.*s", (int)(len - 3), de->d_name); + snprintf(name, sizeof(name), "%.*s", (int)(len - 3), de->name); char clean[64]; cst_tool_sanitize_name(name, clean, sizeof(clean)); int at = find_role(v, *n, clean); @@ -166,14 +163,7 @@ static void load_dir(const char *dir, const char *source, cst_agent_role_t *v, r->prompt = strdup(body); free(text); } - closedir(d); -#else - (void)dir; - (void)source; - (void)v; - (void)n; - (void)max; -#endif + os_dir_close(d); } int cst_roles_list(const char *ws, cst_agent_role_t *out, int max) { @@ -255,18 +245,16 @@ int cst_subagent_path(const char *ws, const char *parent_id, const char *file, static int next_file(const char *dir, char *file, size_t fsz) { pthread_mutex_lock(&g_seq_mu); int best = 0; -#ifndef _WIN32 - DIR *d = opendir(dir); + os_dir_t *d = os_dir_open(dir); if (d) { - struct dirent *de; - while ((de = readdir(d)) != NULL) { + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { int k = 0; - if (sscanf(de->d_name, "sub-%d.jsonl", &k) == 1 && k > best) + if (sscanf(de->name, "sub-%d.jsonl", &k) == 1 && k > best) best = k; } - closedir(d); + os_dir_close(d); } -#endif int rc = -1; for (int k = best + 1; k < best + 100; k++) { char p[OS_PATH_MAX]; @@ -276,7 +264,7 @@ static int next_file(const char *dir, char *file, size_t fsz) { struct stat st; if (stat(p, &st) == 0) continue; - FILE *fp = fopen(p, "a"); + FILE *fp = fopen(p, "ab"); if (fp) { fclose(fp); rc = 0; @@ -302,7 +290,7 @@ static void write_meta(const char *dir, const char *file, const char *role, cJSON_AddNumberToObject(o, "ms", ms); char *txt = cJSON_PrintUnformatted(o); cJSON_Delete(o); - FILE *fp = txt ? fopen(p, "w") : NULL; + FILE *fp = txt ? fopen(p, "wb") : NULL; if (fp) { fputs(txt, fp); fputc('\n', fp); @@ -317,19 +305,18 @@ int cst_subagent_list(const char *ws, const char *parent_id, if (sub_dir(ws, parent_id, dir, sizeof(dir)) != 0) return 0; int n = 0; -#ifndef _WIN32 - DIR *d = opendir(dir); + os_dir_t *d = os_dir_open(dir); if (!d) return 0; - struct dirent *de; - while ((de = readdir(d)) != NULL && n < max) { + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL && n < max) { int k = 0; char tail[8] = ""; - if (sscanf(de->d_name, "sub-%d.%7s", &k, tail) != 2 || + if (sscanf(de->name, "sub-%d.%7s", &k, tail) != 2 || strcmp(tail, "json") != 0) continue; char p[OS_PATH_MAX]; - if (snprintf(p, sizeof(p), "%s/%s", dir, de->d_name) >= (int)sizeof(p)) + if (snprintf(p, sizeof(p), "%s/%s", dir, de->name) >= (int)sizeof(p)) continue; char *txt = slurp(p, 64 * 1024); cJSON *o = txt ? cJSON_Parse(txt) : NULL; @@ -358,11 +345,7 @@ int cst_subagent_list(const char *ws, const char *parent_id, r->mtime = (long long)st.st_mtime; n++; } - closedir(d); -#else - (void)out; - (void)max; -#endif + os_dir_close(d); /* newest first: higher number */ for (int i = 1; i < n; i++) for (int j = i; j > 0 && atoi(out[j - 1].file + 4) < atoi(out[j].file + 4); diff --git a/src/tools/fs_tools.c b/src/tools/fs_tools.c index 8189329..e28ab86 100644 --- a/src/tools/fs_tools.c +++ b/src/tools/fs_tools.c @@ -12,12 +12,7 @@ #include <string.h> #include <sys/stat.h> -#ifndef _WIN32 -#include <dirent.h> -#include <fnmatch.h> #include <regex.h> -#include <unistd.h> -#endif #include "cst_tools.h" #include "cst_util.h" @@ -154,6 +149,9 @@ int cst_tool_read_file2(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, FILE *fp = fopen(full, "rb"); if (!fp) return tool_set_resultf(r, 1, "error: cannot open file: %.200s", path); + char bom[3]; + if (fread(bom, 1, 3, fp) != 3 || !cst_bom_len(bom, 3)) + rewind(fp); long offset = tool_arg_long(c, "offset", 0); long limit = tool_arg_long(c, "limit", 0); @@ -178,6 +176,8 @@ int cst_tool_read_file2(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, return tool_set_resultf(r, 1, "error: %.200s is a binary file (%zu bytes)", path, n); } + cst_strip_bom(buf, &n); + n = cst_crlf_to_lf(buf, n); /* edits match LF text and keep CRLF */ cst_tool_result_free(r); r->out = buf; r->out_len = n; @@ -217,6 +217,9 @@ int cst_tool_read_file2(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, } if (numbers) cst_buf_printf(&b, "%6ld|", lineno); + if (line.len >= 2 && line.buf[line.len - 2] == '\r' && + line.buf[line.len - 1] == '\n') + line.buf[--line.len - 1] = '\n'; cst_buf_append(&b, line.buf, line.len); shown++; } @@ -442,6 +445,14 @@ static int plan_edit(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, nwn = n2n; } } + } else if (cnt && memchr(nw, '\n', nwn) && + cst_text_is_crlf(pl->oldc, pl->olen)) { + size_t n2n; + n2 = to_crlf(nw, nwn, &n2n); + if (n2) { + nw = n2; + nwn = n2n; + } } int rc = 0; if (cnt == 0) { @@ -539,7 +550,6 @@ int cst_preview_edit_file(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, /* --- delete_file / move_file --------------------------------------------- */ static int reject_symlink(const cst_tool_ctx_t *ctx, const char *path) { -#ifndef _WIN32 char root[OS_PATH_MAX], joined[OS_PATH_MAX]; if (os_realpath(ctx->workspace_root, root, sizeof(root)) != 0) return 1; @@ -547,13 +557,9 @@ static int reject_symlink(const cst_tool_ctx_t *ctx, const char *path) { snprintf(joined, sizeof(joined), "%s", path); else if (os_path_join(joined, sizeof(joined), root, path) != 0) return 1; - struct stat st; - if (lstat(joined, &st) == 0 && S_ISLNK(st.st_mode)) + os_stat_t st; + if (os_lstat(joined, &st) == 0 && st.is_link) return 1; -#else - (void)ctx; - (void)path; -#endif return 0; } @@ -672,8 +678,6 @@ int cst_tool_move_file(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, /* --- directory walking (list_dir / glob / grep) -------------------------- */ -#ifndef _WIN32 - static const char *const SKIP_DIRS[] = { ".git", ".hg", ".svn", "node_modules", ".agent", "__pycache__", ".venv", ".cache", ".idea", ".vscode", ".mypy_cache", ".pytest_cache"}; @@ -703,7 +707,7 @@ static void ignore_load(const char *root, ignore_t *ig) { char p[OS_PATH_MAX]; if (os_path_join(p, sizeof(p), root, ".gitignore") != 0) return; - FILE *fp = fopen(p, "r"); + FILE *fp = fopen(p, "rb"); if (!fp) return; char line[512]; @@ -749,7 +753,7 @@ static int ignored(const ignore_t *ig, const char *rel, const char *name, const ign_rule_t *r = &ig->v[i]; if (r->dir_only && !isdir) continue; - if (fnmatch(r->pat, r->anchored ? rel : name, 0) == 0) + if (cst_fnmatch(r->pat, r->anchored ? rel : name, 0) == 0) return 1; } return 0; @@ -758,7 +762,7 @@ static int ignored(const ignore_t *ig, const char *rel, const char *name, typedef struct walker walker_t; /* Return non-zero to stop the walk. */ typedef int (*walk_fn)(walker_t *w, const char *rel, const char *full, - const struct stat *st, int depth); + const os_stat_t *st, int depth); struct walker { const char *root; /* workspace realpath */ @@ -781,15 +785,13 @@ static void walk_dir(walker_t *w, const char *rel, int depth) { snprintf(dirfull, sizeof(dirfull), "%s/%s", w->root, rel); else snprintf(dirfull, sizeof(dirfull), "%s", w->root); - DIR *d = opendir(dirfull); + os_dir_t *d = os_dir_open(dirfull); if (!d) return; char **names = NULL; int n = 0, cap = 0; - struct dirent *de; - while ((de = readdir(d)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) - continue; + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { if (n == cap) { cap = cap ? cap * 2 : 32; char **nv = realloc(names, (size_t)cap * sizeof(char *)); @@ -797,11 +799,11 @@ static void walk_dir(walker_t *w, const char *rel, int depth) { break; names = nv; } - names[n] = strdup(de->d_name); + names[n] = strdup(de->name); if (names[n]) n++; } - closedir(d); + os_dir_close(d); qsort(names, (size_t)n, sizeof(char *), name_cmp); for (int i = 0; i < n && !w->stop; i++) { char crel[OS_PATH_MAX], cfull[OS_PATH_MAX]; @@ -810,10 +812,10 @@ static void walk_dir(walker_t *w, const char *rel, int depth) { int k2 = snprintf(cfull, sizeof(cfull), "%s/%s", w->root, crel); if (k1 >= (int)sizeof(crel) || k2 >= (int)sizeof(cfull)) continue; /* path too long */ - struct stat st; - if (lstat(cfull, &st) != 0) + os_stat_t st; + if (os_lstat(cfull, &st) != 0) continue; - int isdir = S_ISDIR(st.st_mode); + int isdir = st.is_dir; if (ignored(&w->ig, crel, names[i], isdir)) continue; if (w->fn(w, crel, cfull, &st, depth)) @@ -861,21 +863,18 @@ typedef struct { /* Entries directly inside a directory (for the depth boundary). */ static int dir_entries(const char *full) { - DIR *d = opendir(full); + os_dir_t *d = os_dir_open(full); if (!d) return -1; int n = 0; - struct dirent *de; - while ((de = readdir(d)) != NULL) { - if (strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) - n++; - } - closedir(d); + while (os_dir_next(d)) + n++; + os_dir_close(d); return n; } static int list_cb(walker_t *w, const char *rel, const char *full, - const struct stat *st, int depth) { + const os_stat_t *st, int depth) { list_ud_t *u = w->ud; if (u->count >= LIST_MAX) { if (++u->more >= LIST_COUNT_MAX) { @@ -888,7 +887,7 @@ static int list_cb(walker_t *w, const char *rel, const char *full, name = name ? name + 1 : rel; for (int i = 0; i < depth; i++) cst_buf_puts(&u->b, " "); - if (S_ISDIR(st->st_mode)) { + if (st->is_dir) { int k = -1; if (skip_dir_name(name)) cst_buf_printf(&u->b, "%s/ (skipped)\n", name); @@ -897,10 +896,10 @@ static int list_cb(walker_t *w, const char *rel, const char *full, k == 1 ? "y" : "ies"); else cst_buf_printf(&u->b, "%s/%s\n", name, k == 0 ? " (empty)" : ""); - } else if (S_ISLNK(st->st_mode)) + } else if (st->is_link) cst_buf_printf(&u->b, "%s@\n", name); else - cst_buf_printf(&u->b, "%s (%lld B)\n", name, (long long)st->st_size); + cst_buf_printf(&u->b, "%s (%lld B)\n", name, st->size); u->count++; return 0; } @@ -974,7 +973,7 @@ static int seg_match(const char *pat, const char *path) { pseg[pl] = 0; memcpy(sseg, path, sl); sseg[sl] = 0; - if (fnmatch(pseg, sseg, FNM_PERIOD) != 0) + if (cst_fnmatch(pseg, sseg, CST_FNM_PERIOD) != 0) return 0; if (!ps) return !ss; @@ -994,11 +993,11 @@ typedef struct { } glob_ud_t; static int glob_cb(walker_t *w, const char *rel, const char *full, - const struct stat *st, int depth) { + const os_stat_t *st, int depth) { (void)full; (void)depth; glob_ud_t *u = w->ud; - if (!S_ISREG(st->st_mode)) + if (!st->is_reg) return 0; const char *sub = rel; size_t bl = strlen(u->base); @@ -1009,7 +1008,7 @@ static int glob_cb(walker_t *w, const char *rel, const char *full, } const char *name = strrchr(sub, '/'); name = name ? name + 1 : sub; - int ok = u->basename_only ? fnmatch(u->pattern, name, FNM_PERIOD) == 0 + int ok = u->basename_only ? cst_fnmatch(u->pattern, name, CST_FNM_PERIOD) == 0 : seg_match(u->pattern, sub); if (!ok) return 0; @@ -1025,7 +1024,7 @@ static int glob_cb(walker_t *w, const char *rel, const char *full, return 1; } u->hits[u->n] = strdup(rel); - u->mtimes[u->n] = (double)st->st_mtime; + u->mtimes[u->n] = st->mtime; if (u->hits[u->n]) u->n++; if (u->n >= GLOB_MAX * 20) { @@ -1109,11 +1108,11 @@ typedef struct { } files_ud_t; static int files_cb(walker_t *w, const char *rel, const char *full, - const struct stat *st, int depth) { + const os_stat_t *st, int depth) { (void)full; (void)depth; files_ud_t *u = w->ud; - if (!S_ISREG(st->st_mode)) + if (!st->is_reg) return 0; if (u->n >= u->max) return 1; @@ -1275,16 +1274,16 @@ static void grep_file(grep_ud_t *u, const char *rel, const char *full) { } static int grep_cb(walker_t *w, const char *rel, const char *full, - const struct stat *st, int depth) { + const os_stat_t *st, int depth) { (void)depth; grep_ud_t *u = w->ud; - if (!S_ISREG(st->st_mode) || st->st_size > READ_FILE_MAX) + if (!st->is_reg || st->size > READ_FILE_MAX) return 0; if (u->glob) { const char *name = strrchr(rel, '/'); name = name ? name + 1 : rel; - int ok = strchr(u->glob, '/') ? fnmatch(u->glob, rel, 0) == 0 - : fnmatch(u->glob, name, 0) == 0; + int ok = strchr(u->glob, '/') ? cst_fnmatch(u->glob, rel, 0) == 0 + : cst_fnmatch(u->glob, name, 0) == 0; if (!ok) return 0; } @@ -1391,32 +1390,4 @@ int cst_tool_grep(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, return tool_take_buf(r, &out, 0); } -#else /* _WIN32: directory tools are not implemented natively */ -int cst_tool_list_dir(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, - cst_tool_result_t *r) { - (void)ctx; - (void)c; - return tool_set_result(r, "error: list_dir is not available on Windows; " - "use run_shell", 1); -} -int cst_workspace_files(const char *ws, char ***out, int max) { - (void)ws; - (void)max; - *out = NULL; - return 0; -} -int cst_tool_glob(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, - cst_tool_result_t *r) { - (void)ctx; - (void)c; - return tool_set_result(r, "error: glob is not available on Windows", 1); -} -int cst_tool_grep(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, - cst_tool_result_t *r) { - (void)ctx; - (void)c; - return tool_set_result(r, "error: grep is not available on Windows", 1); -} - -#endif diff --git a/src/tools/patch.c b/src/tools/patch.c index b815757..1fc124b 100644 --- a/src/tools/patch.c +++ b/src/tools/patch.c @@ -1,4 +1,5 @@ #include "cst_agent.h" +#include "cst_util.h" #include <stdio.h> #include <stdlib.h> @@ -129,9 +130,9 @@ static int parse_hunk_header(const char *p, size_t n, long *a, long *b, return -1; } -int cst_patch_apply_buffer(const char *orig, size_t olen, const char *hunks, - size_t hlen, char **out, size_t *out_len, - int *nhunks, char *err, size_t err_sz) { +static int patch_core(const char *orig, size_t olen, const char *hunks, + size_t hlen, char **out, size_t *out_len, int *nhunks, + char *err, size_t err_sz) { lines_t file = {0}, hl = {0}; lines_t old_seq = {0}, new_seq = {0}; int rc = -1; @@ -315,3 +316,47 @@ done: free(new_seq.v); return rc; } + +/* A CRLF file is patched as LF (the model's hunks usually are) and written + * back as CRLF; a BOM is kept out of the first line's context. */ +int cst_patch_apply_buffer(const char *orig, size_t olen, const char *hunks, + size_t hlen, char **out, size_t *out_len, + int *nhunks, char *err, size_t err_sz) { + size_t bom = orig ? cst_bom_len(orig, olen) : 0; + int crlf = orig && cst_text_is_crlf(orig, olen); + if (!bom && !crlf) + return patch_core(orig, olen, hunks, hlen, out, out_len, nhunks, err, + err_sz); + if (out) + *out = NULL; + char *o = malloc(olen - bom + 1), *h = malloc(hlen + 1), *res = NULL; + size_t rlen = 0; + int rc = -1; + if (!o || !h) { + snprintf(err, err_sz, "oom"); + goto done; + } + memcpy(o, orig + bom, olen - bom); + o[olen - bom] = 0; + size_t on = olen - bom, hn = hlen; + if (hlen) + memcpy(h, hunks, hlen); + h[hlen] = 0; + if (crlf) { + on = cst_crlf_to_lf(o, on); + hn = cst_crlf_to_lf(h, hn); + } + rc = patch_core(o, on, h, hn, &res, &rlen, nhunks, err, err_sz); + if (rc == 0) { + *out = cst_text_restyle(res, rlen, crlf, bom != 0, out_len); + if (!*out) { + snprintf(err, err_sz, "oom"); + rc = -1; + } + } +done: + free(res); + free(o); + free(h); + return rc; +} diff --git a/src/tools/registry.c b/src/tools/registry.c index c87996b..9f056ce 100644 --- a/src/tools/registry.c +++ b/src/tools/registry.c @@ -6,9 +6,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> -#ifndef _WIN32 #include <pthread.h> -#endif #include "cJSON.h" #include "cst_tools.h" @@ -182,14 +180,9 @@ const char *cst_tool_profile_name(int p) { #define DYN_MAX 256 static cst_tool_def_t *g_dyn[DYN_MAX]; static atomic_int g_ndyn; -#ifndef _WIN32 static pthread_mutex_t g_dyn_mu = PTHREAD_MUTEX_INITIALIZER; #define DYN_LOCK() pthread_mutex_lock(&g_dyn_mu) #define DYN_UNLOCK() pthread_mutex_unlock(&g_dyn_mu) -#else -#define DYN_LOCK() ((void)0) -#define DYN_UNLOCK() ((void)0) -#endif int cst_tool_count(void) { return NTOOLS + atomic_load(&g_ndyn); } diff --git a/src/tools/tools.c b/src/tools/tools.c index a3079ca..9c33f1e 100644 --- a/src/tools/tools.c +++ b/src/tools/tools.c @@ -238,6 +238,22 @@ static int write_content(const ctl_tool_call_t *call, const char **content, return *content ? 0 : -1; } +/* Overwriting a CRLF and/or BOM file with plain LF text keeps the file's + * style: malloc'd restyled copy, or NULL to write content as given. */ +static char *keep_text_style(const char *oldc, size_t olen, const char *s, + size_t n, size_t *out_len) { + if (!oldc) + return NULL; + int has_crlf = 0; + for (size_t i = 1; i < n && !has_crlf; i++) + has_crlf = s[i] == '\n' && s[i - 1] == '\r'; + int crlf = !has_crlf && cst_text_is_crlf(oldc, olen); + int bom = cst_bom_len(oldc, olen) && !cst_bom_len(s, n); + if (!crlf && !bom) + return NULL; + return cst_text_restyle(s, n, crlf, 1, out_len); +} + int cst_preview_write_file(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, char **out) { const char *path = tool_arg(c, "path"); @@ -252,11 +268,18 @@ int cst_preview_write_file(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, char *oldc = NULL; if (cst_sandbox_resolve(ctx->workspace_root, path, full, sizeof(full)) == 0) oldc = tool_slurp(full, &olen, 0); + size_t sn = 0; + char *styled = keep_text_style(oldc, olen, content, n, &sn); + if (styled) { + content = styled; + n = sn; + } cst_buf_t b = {0}; cst_buf_printf(&b, "%s %s (%zu bytes)\n", oldc ? "overwrite" : "create", path, n); tool_append_diff(&b, path, oldc, olen, content, n); free(oldc); + free(styled); *out = cst_buf_steal(&b); return 0; } @@ -276,14 +299,22 @@ int cst_tool_write_file(const char *workspace_root, const ctl_tool_call_t *call, size_t olen = 0; char *oldc = tool_slurp(full, &olen, 0); + size_t sn = 0; + char *styled = keep_text_style(oldc, olen, content, n, &sn); + if (styled) { + content = styled; + n = sn; + } if (tool_write_whole(full, content, n) != 0) { free(oldc); + free(styled); return tool_set_resultf(r, 1, "error: cannot write file: %.200s", path); } cst_buf_t b = {0}; cst_buf_printf(&b, "wrote %s (%zu bytes)\n", path, n); tool_append_diff(&b, path, oldc, olen, content, n); free(oldc); + free(styled); return tool_take_buf(r, &b, 0); } diff --git a/src/tui.c b/src/tui.c index c58c4a6..ae63e70 100644 --- a/src/tui.c +++ b/src/tui.c @@ -41,11 +41,12 @@ static int g_nocolor; /* NO_COLOR */ static int g_ascii; /* non-UTF-8 locale or CST_ASCII=1 */ +static int g_logo_text; /* console font lacks the logo's block glyphs */ static int g_utf8 = 1; /* locale can show non-ASCII text */ #define HAS_COLOR() (!g_nocolor && has_colors()) -/* Curses: ncursesw on Unix, PDCurses (wide) on Windows. */ -#if defined(CST_USE_PDCURSES) || defined(_WIN32) +/* Curses: ncursesw (Unix and Windows), or PDCurses (wide) on Windows. */ +#if defined(CST_USE_PDCURSES) #ifndef PDC_WIDE #define PDC_WIDE 1 #endif @@ -62,10 +63,23 @@ static int g_utf8 = 1; /* locale can show non-ASCII text */ #ifndef KEY_RESIZE #define KEY_RESIZE 0632 #endif -#define wcwidth cst_wcwidth /* MinGW often lacks wcwidth */ #else #include <ncurses.h> #endif +#ifdef _WIN32 +#define wcwidth cst_wcwidth /* MinGW often lacks wcwidth */ +#endif + +/* The Windows console (conhost/ConPTY) loses wide characters on insert or + * delete-character, so curses must repaint cells instead. */ +static WINDOW *tui_newwin(int rows, int cols, int y, int x) { + WINDOW *w = newwin(rows, cols, y, x); +#ifdef _WIN32 + if (w) + idcok(w, FALSE); +#endif + return w; +} #ifdef _WIN32 static void cst_usleep(unsigned usec) { @@ -1876,10 +1890,10 @@ static void git_branch(const char *dir, char *out, size_t sz) { FILE *f = NULL; for (int depth = 0; depth < 64 && d[0]; depth++) { snprintf(p, sizeof(p), "%s/.git/HEAD", d); - if ((f = fopen(p, "r")) != NULL) + if ((f = fopen(p, "rb")) != NULL) break; snprintf(p, sizeof(p), "%s/.git", d); - FILE *g = fopen(p, "r"); + FILE *g = fopen(p, "rb"); if (g) { int ok = fgets(line, sizeof(line), g) && strncmp(line, "gitdir: ", 8) == 0; fclose(g); @@ -1889,7 +1903,7 @@ static void git_branch(const char *dir, char *out, size_t sz) { ? snprintf(p, sizeof(p), "%s/HEAD", line + 8) : snprintf(p, sizeof(p), "%s/%s/HEAD", d, line + 8); if (n > 0 && (size_t)n < sizeof(p)) - f = fopen(p, "r"); + f = fopen(p, "rb"); break; } } @@ -2322,7 +2336,7 @@ static void draw_home(tui_t *t) { getmaxyx(t->logw, h, w); int box_top = t->in_y - t->log_y; int box_bot = box_top + t->in_h; - if (g_ascii || !g_utf8) { + if (g_ascii || !g_utf8 || g_logo_text) { int x = (w - 8) / 2, y = box_top - 2; if (y >= 0) { draw_ansi_at(t->logw, y, x, 3, "cst", CP_LOGO_DIM, 1, 0); @@ -2805,12 +2819,107 @@ static int input_visual_rows(const tui_t *t) { return rows; } +#ifdef _WIN32 +static DWORD g_con_in_mode; +static int g_con_in_saved; + +/* QuickEdit makes the console select text itself and swallow the clicks; + * it is off while the TUI owns the mouse and restored on exit. */ +static void win_console_mouse(int on) { + HANDLE h = GetStdHandle(STD_INPUT_HANDLE); + DWORD m; + if (!GetConsoleMode(h, &m)) + return; + if (!g_con_in_saved) { + g_con_in_mode = m; + g_con_in_saved = 1; + } + if (on) + m = (m & ~(DWORD)ENABLE_QUICK_EDIT_MODE) | ENABLE_MOUSE_INPUT | + ENABLE_EXTENDED_FLAGS; + else + m = g_con_in_mode | ENABLE_EXTENDED_FLAGS; + SetConsoleMode(h, m); +} + +static CONSOLE_FONT_INFOEX g_con_font; +static int g_con_font_saved; + +/* conhost has no glyph fallback: NSimSun, the default on Chinese systems, + * draws U+2580 as an empty box. */ +static int win_font_has(const WCHAR *face, WCHAR ch) { + HDC dc = CreateCompatibleDC(NULL); + if (!dc) + return 1; + HFONT font = CreateFontW(16, 0, 0, 0, FW_NORMAL, 0, 0, 0, DEFAULT_CHARSET, + 0, 0, 0, FF_MODERN, face); + WORD idx = 0; + int ok = 1; + if (font) { + HGDIOBJ old = SelectObject(dc, font); + if (GetGlyphIndicesW(dc, &ch, 1, &idx, GGI_MARK_NONEXISTING_GLYPHS) == 1) + ok = idx != 0xffff; + SelectObject(dc, old); + DeleteObject(font); + } + DeleteDC(dc); + return ok; +} + +/* Raster fonts cannot draw UTF-8 box drawing or CJK; switch the console to + * a TrueType font of the same height while the TUI runs. */ +static void win_console_font(int on) { + HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); + if (!on) { + if (g_con_font_saved) + SetCurrentConsoleFontEx(h, FALSE, &g_con_font); + g_con_font_saved = 0; + return; + } + CONSOLE_FONT_INFOEX f = {.cbSize = sizeof f}; + if (!GetCurrentConsoleFontEx(h, FALSE, &f)) + return; + if (!(f.FontFamily & TMPF_TRUETYPE)) { + g_con_font = f; + CONSOLE_FONT_INFOEX tt = {.cbSize = sizeof tt}; + tt.dwFontSize.Y = f.dwFontSize.Y > 0 ? f.dwFontSize.Y : 16; + tt.FontFamily = FF_MODERN | TMPF_TRUETYPE; + tt.FontWeight = FW_NORMAL; + wcscpy(tt.FaceName, GetACP() == 936 ? L"NSimSun" : L"Consolas"); + if (!SetCurrentConsoleFontEx(h, FALSE, &tt)) + return; + g_con_font_saved = 1; + f = tt; + } + if (!getenv("WT_SESSION") && !win_font_has(f.FaceName, 0x2580)) + g_logo_text = 1; +} +#endif + +#define MOUSE_RUN_MAX 16 +/* getmouse() hands out a run of queued events newest first (the Windows + * console driver queues press, motion and release together); returns them + * oldest first. */ +static int get_mouse_run(MEVENT *out, int max) { + int n = 0; + while (n < max && getmouse(&out[n]) == OK) + n++; + for (int i = 0; i < n / 2; i++) { + MEVENT tmp = out[i]; + out[i] = out[n - 1 - i]; + out[n - 1 - i] = tmp; + } + return n; +} + static void apply_mouse(const tui_t *t) { mousemask(t->mouse_on ? (mmask_t)(ALL_MOUSE_EVENTS | REPORT_MOUSE_POSITION) : 0, NULL); mouseinterval(0); -#ifdef NCURSES_VERSION +#ifdef _WIN32 + win_console_mouse(t->mouse_on); +#elif defined(NCURSES_VERSION) /* ncurses only asks for presses and releases (1000). 1002 also reports * motion while a button is held, for drag selection; terminals treat it as * replacing 1000, so it goes after mousemask and is turned off explicitly. */ @@ -2869,8 +2978,8 @@ static void layout(tui_t *t) { int top = (t->rows - 1 - block) / 2; t->in_y = clampi(top + LOGO_H + 1, 0, t->rows - 1 - t->in_h); t->log_x = t->log_y = 0; - t->logw = newwin(t->rows - 1, t->cols, 0, 0); - t->status = newwin(1, t->cols - 2, t->rows - 1, 1); + t->logw = tui_newwin(t->rows - 1, t->cols, 0, 0); + t->status = tui_newwin(1, t->cols - 2, t->rows - 1, 1); } else { int mw = t->cols - t->side_w; t->in_y = t->rows - 1 - t->in_h; @@ -2885,14 +2994,14 @@ static void layout(tui_t *t) { t->todo_rows = 0; log_h = t->in_y - t->log_y > 0 ? t->in_y - t->log_y : 1; } - t->logw = newwin(log_h, mw - 2, t->log_y, 1); + t->logw = tui_newwin(log_h, mw - 2, t->log_y, 1); if (t->todo_rows) - t->todow = newwin(t->todo_rows, mw - 2, t->log_y + log_h, 1); - t->status = newwin(1, mw - 2, t->rows - 1, 1); + t->todow = tui_newwin(t->todo_rows, mw - 2, t->log_y + log_h, 1); + t->status = tui_newwin(1, mw - 2, t->rows - 1, 1); if (t->side_w) - t->side = newwin(t->rows, t->side_w, 0, mw); + t->side = tui_newwin(t->rows, t->side_w, 0, mw); } - t->input = newwin(t->in_h, t->in_w, t->in_y, t->in_x); + t->input = tui_newwin(t->in_h, t->in_w, t->in_y, t->in_x); if (HAS_COLOR()) { wbkgdset(t->logw, COLOR_PAIR(CP_PANEL)); wbkgdset(t->status, COLOR_PAIR(CP_PANEL)); @@ -3495,7 +3604,7 @@ static int picker_run(tui_t *t, const char *title, char items[][128], int n, redraw_all(t); /* uncover what a larger dialog hid */ last_dh = dh; last_dw = dw; - dlg = newwin(dh, dw, dy, dx); + dlg = tui_newwin(dh, dw, dy, dx); if (!dlg) break; keypad(dlg, TRUE); @@ -3549,9 +3658,14 @@ static int picker_run(tui_t *t, const char *title, char items[][128], int n, layout(t); redraw_all(t); } else if (wch == KEY_MOUSE) { - MEVENT ev; - if (getmouse(&ev) != OK) + MEVENT evs[MOUSE_RUN_MAX], ev; + int ne = get_mouse_run(evs, MOUSE_RUN_MAX), k0 = ne - 1; + if (ne == 0) continue; + for (int i = 0; i < ne; i++) + if (evs[i].bstate & (BUTTON1_PRESSED | BUTTON1_CLICKED)) + k0 = i; /* a click anywhere in the run wins over its release */ + ev = evs[k0]; if (ev.bstate & BUTTON4_PRESSED) { if (sel > 0) sel--; @@ -4301,6 +4415,34 @@ static void cmd_tools(tui_t *t, char *arg, int *quit) { } } +static void cmd_sandbox(tui_t *t, char *arg, int *quit) { + (void)quit; + if (arg && strcmp(arg, "reset-labels") == 0) { + char *rep = NULL; + int n = cst_sandbox_reset_labels(&rep); + if (rep && rep[0]) + push_lines(t, " ", rep); + push_system_line(t, n < 0 ? "(some labels could not be restored; " + "they stay listed)" + : n == 0 ? "(no labeled directories)" + : "(labels restored)"); + free(rep); + return; + } + int sk = cst_sandbox_probe(); + push_system_line(t, "Shell sandbox: %s", + sk == CST_SANDBOX_NONE ? "none (rule-based approval)" + : cst_sandbox_name(sk)); + char *l = cst_sandbox_labels(); + if (l) { + push_system_line(t, "Labeled Low integrity (writable by sandboxed " + "commands):"); + push_lines(t, " ", l); + push_system_line(t, "/sandbox reset-labels restores their old labels"); + free(l); + } +} + static void cmd_permissions(tui_t *t, char *arg, int *quit) { (void)quit; cst_perms_t *p = cst_agent_perms(t->agent); @@ -4537,7 +4679,7 @@ static void cmd_export(tui_t *t, char *arg, int *quit) { *sl = 0; os_mkdir_p(dir, 0755); } - FILE *fp = fopen(path, "w"); + FILE *fp = fopen(path, "wb"); if (!fp) { push_system_line(t, "Error: cannot write %s", path); return; @@ -4569,9 +4711,19 @@ static void cmd_export(tui_t *t, char *arg, int *quit) { push_system_line(t, "(exported to %s)", path); } +#ifdef _WIN32 +#define COPY_VIA "clipboard" +#else +#define COPY_VIA "OSC 52" +#endif + /* Puts text on the terminal's clipboard; returns the bytes sent, 0 on * failure. */ static size_t osc52_copy(const char *text, size_t n) { +#ifdef _WIN32 + if (os_clipboard_set(text, n) == 0) + return n; /* conhost has no OSC 52 */ +#endif static const char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; if (n > 75000) @@ -4614,8 +4766,8 @@ static void cmd_copy(tui_t *t, char *arg, int *quit) { } size_t n = osc52_copy(text, strlen(text)); if (n) - push_system_line(t, "(copied %zu bytes of the last reply via OSC 52 — the " - "terminal must allow clipboard access)", n); + push_system_line(t, "(copied %zu bytes of the last reply via " COPY_VIA + " — the terminal must allow clipboard access)", n); } static void cmd_init(tui_t *t, char *arg, int *quit) { @@ -4945,6 +5097,8 @@ static const slash_cmd_t SLASH[] = { cmd_tools}, {"permissions", "perms", "[reset]", "show saved always-allow rules", 1, cmd_permissions}, + {"sandbox", "", "[reset-labels]", "shell sandbox and labeled folders", 1, + cmd_sandbox}, {"approval", "", "[ask|auto|yolo]", "how tools get approved (Shift+Tab)", 1, cmd_approval}, {"yolo", "", "", "toggle approving everything", 1, cmd_yolo}, @@ -5530,7 +5684,7 @@ static void sel_copy(tui_t *t) { for (size_t i = 0; i < b.n; i++) chars += ((unsigned char)b.s[i] & 0xC0) != 0x80; char msg[96]; - snprintf(msg, sizeof(msg), "Copied %zu characters (OSC 52)", chars); + snprintf(msg, sizeof(msg), "Copied %zu characters (" COPY_VIA ")", chars); set_status(t, msg); draw_status(t); } @@ -6187,6 +6341,21 @@ static void insert_newline(tui_t *t) { } /* Paste mode: keep newlines/tabs literally; redraw once at the end. */ +#ifdef _WIN32 +static int win_key_pending(void) { + INPUT_RECORD rec[32]; + DWORD n = 0; + HANDLE h = GetStdHandle(STD_INPUT_HANDLE); + if (!PeekConsoleInputW(h, rec, 32, &n)) + return 0; + for (DWORD i = 0; i < n; i++) + if (rec[i].EventType == KEY_EVENT && rec[i].Event.KeyEvent.bKeyDown && + rec[i].Event.KeyEvent.uChar.UnicodeChar) + return 1; + return 0; +} +#endif + static void paste_char(tui_t *t, wint_t wch) { if (wch == 13) { input_put_byte(t, '\n'); @@ -6207,20 +6376,31 @@ static void paste_char(tui_t *t, wint_t wch) { } int cst_tui_run(cst_agent_t *agent) { +#ifdef _WIN32 + setlocale(LC_ALL, ".UTF-8"); +#else if (!setlocale(LC_ALL, "")) setlocale(LC_ALL, "C.UTF-8"); +#endif #ifndef _WIN32 if (!getenv("ESCDELAY")) setenv("ESCDELAY", "25", 0); #endif +#ifdef _WIN32 + win_console_font(1); +#endif initscr(); +#ifdef _WIN32 + idcok(stdscr, FALSE); +#endif raw(); /* Ctrl+C/Ctrl+Z reach us as keys instead of signals */ noecho(); keypad(stdscr, TRUE); curs_set(1); theme_init(); -#ifdef NCURSES_VERSION +#if defined(NCURSES_VERSION) && !defined(_WIN32) + /* the Windows console driver reads key events, not VT sequences */ define_key("\033[200~", KEY_PASTE_BEGIN); define_key("\033[201~", KEY_PASTE_END); printf("\033[?2004h"); @@ -6375,6 +6555,18 @@ int cst_tui_run(cst_agent_t *agent) { if (t.sel_state == SEL_SHOWN && !(rc == KEY_CODE_YES && (wch == KEY_MOUSE || wch == KEY_RESIZE))) sel_clear(&t); +#ifdef _WIN32 + /* The console has no bracketed paste: an Enter with more keys already + * queued is a pasted line break, not a send. */ + if (rc == OK && !t.in_paste && (wch == 10 || wch == 13) && + win_key_pending()) { + paste_char(&t, wch); + input_changed(&t); + after_edit(&t); + draw_input(&t); + continue; + } +#endif if (!t.in_paste && !(rc == KEY_CODE_YES && @@ -6453,12 +6645,13 @@ int cst_tui_run(cst_agent_t *agent) { continue; } if (wch == KEY_MOUSE) { - MEVENT ev; - if (getmouse(&ev) == OK) { - if (on_mouse(&t, &ev, busy) == -2) + MEVENT evs[MOUSE_RUN_MAX]; + int ne = get_mouse_run(evs, MOUSE_RUN_MAX); + for (int i = 0; i < ne; i++) + if (on_mouse(&t, &evs[i], busy) == -2) goto out; + if (ne) draw_input(&t); /* keep the terminal cursor in the box */ - } continue; } int wdir; @@ -6667,7 +6860,11 @@ out: usleep(100000); /* let a /models probe finish before curl cleanup */ tui_free(&t); -#ifdef NCURSES_VERSION +#ifdef _WIN32 + if (g_con_in_saved) + win_console_mouse(0); + win_console_font(0); +#elif defined(NCURSES_VERSION) kitty_restore(); printf("\033[?1002l\033[?2004l"); fflush(stdout); diff --git a/src/undo.c b/src/undo.c index a8ce87d..62ae104 100644 --- a/src/undo.c +++ b/src/undo.c @@ -9,10 +9,7 @@ #include <string.h> #include <sys/stat.h> -#ifndef _WIN32 -#include <dirent.h> #include <unistd.h> -#endif #include "cst_agent.h" #include "cst_undo.h" @@ -75,7 +72,7 @@ static int put(const char *p, const char *d, size_t n) { } static int manifest_has(const char *mf, const char *rel) { - FILE *fp = fopen(mf, "r"); + FILE *fp = fopen(mf, "rb"); if (!fp) return 0; char line[1400]; @@ -120,7 +117,7 @@ int cst_undo_snapshot(const char *ws, const char *sid, const char *tag, return 0; int idx = 0; { - FILE *fp = fopen(mf, "r"); + FILE *fp = fopen(mf, "rb"); if (fp) { int ch; while ((ch = fgetc(fp)) != EOF) @@ -144,7 +141,7 @@ int cst_undo_snapshot(const char *ws, const char *sid, const char *tag, free(data); } } - FILE *fp = fopen(mf, "a"); + FILE *fp = fopen(mf, "ab"); if (!fp) return -1; fprintf(fp, "%d\t%s\t%s\n", idx, state, rel); @@ -152,35 +149,34 @@ int cst_undo_snapshot(const char *ws, const char *sid, const char *tag, return 0; } -#ifndef _WIN32 static int latest_tag(const char *root, char *tag, size_t n) { - DIR *d = opendir(root); + os_dir_t *d = os_dir_open(root); if (!d) return -1; - struct dirent *de; + const os_dirent_t *de; tag[0] = 0; - while ((de = readdir(d)) != NULL) { - if (de->d_name[0] == '.') + while ((de = os_dir_next(d)) != NULL) { + if (de->name[0] == '.') continue; - if (strcmp(de->d_name, tag) > 0) - snprintf(tag, n, "%s", de->d_name); + if (strcmp(de->name, tag) > 0) + snprintf(tag, n, "%s", de->name); } - closedir(d); + os_dir_close(d); return tag[0] ? 0 : -1; } static void rm_dir(const char *dir) { - DIR *d = opendir(dir); + os_dir_t *d = os_dir_open(dir); if (d) { - struct dirent *de; - while ((de = readdir(d)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) { + if (strcmp(de->name, ".") == 0 || strcmp(de->name, "..") == 0) continue; char p[OS_PATH_MAX]; - if (snprintf(p, sizeof(p), "%s/%s", dir, de->d_name) < (int)sizeof(p)) + if (snprintf(p, sizeof(p), "%s/%s", dir, de->name) < (int)sizeof(p)) unlink(p); } - closedir(d); + os_dir_close(d); } rmdir(dir); } @@ -189,14 +185,14 @@ int cst_undo_count(const char *ws, const char *sid) { char root[OS_PATH_MAX]; if (undo_root(ws, sid, root, sizeof(root)) != 0) return 0; - DIR *d = opendir(root); + os_dir_t *d = os_dir_open(root); if (!d) return 0; int n = 0; - struct dirent *de; - while ((de = readdir(d)) != NULL) - n += de->d_name[0] != '.'; - closedir(d); + const os_dirent_t *de; + while ((de = os_dir_next(d)) != NULL) + n += de->name[0] != '.'; + os_dir_close(d); return n; } @@ -214,7 +210,7 @@ int cst_undo_restore_last(const char *ws, const char *sid, char *report, snprintf(report, report_sz, "undo path too long"); return -1; } - FILE *fp = fopen(mf, "r"); + FILE *fp = fopen(mf, "rb"); cst_buf_t rep = {0}; int restored = 0, failed = 0; if (fp) { @@ -231,10 +227,10 @@ int cst_undo_restore_last(const char *ws, const char *sid, char *report, *t3++ = 0; const char *rel = t3, *state = t2; char full[OS_PATH_MAX]; - struct stat lst; + os_stat_t lst; if (!rel_ok(rel) || os_path_join(full, sizeof(full), ws, rel) != 0) continue; - if (lstat(full, &lst) == 0 && S_ISLNK(lst.st_mode)) { + if (os_lstat(full, &lst) == 0 && lst.is_link) { cst_buf_printf(&rep, "skipped %s (now a symlink)\n", rel); failed++; continue; @@ -280,17 +276,3 @@ int cst_undo_restore_last(const char *ws, const char *sid, char *report, cst_buf_free(&rep); return failed && !restored ? -1 : 0; } -#else -int cst_undo_count(const char *ws, const char *sid) { - (void)ws; - (void)sid; - return 0; -} -int cst_undo_restore_last(const char *ws, const char *sid, char *report, - size_t report_sz) { - (void)ws; - (void)sid; - snprintf(report, report_sz, "undo is not available on Windows"); - return -1; -} -#endif diff --git a/src/util.c b/src/util.c index 9b72dc7..8f98d59 100644 --- a/src/util.c +++ b/src/util.c @@ -4,6 +4,8 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <wchar.h> +#include <wctype.h> static int buf_reserve(cst_buf_t *b, size_t extra) { if (b->len + extra + 1 <= b->cap) @@ -153,6 +155,213 @@ char *cst_utf8_sanitize(const char *s, size_t n, size_t *out_len) { return out; } +int cst_utf8_valid(const char *s, size_t n) { + const unsigned char *p = (const unsigned char *)s; + for (size_t i = 0; i < n;) { + size_t k = utf8_seq_ok(p + i, n - i); + if (k == 0) + return 0; + i += k; + } + return 1; +} + +size_t cst_bom_len(const char *s, size_t n) { + return n >= 3 && memcmp(s, "\xEF\xBB\xBF", 3) == 0 ? 3 : 0; +} + +void cst_strip_bom(char *s, size_t *len) { + if (!s) + return; + size_t n = len ? *len : strlen(s); + if (!cst_bom_len(s, n)) + return; + memmove(s, s + 3, n - 3 + 1); + if (len) + *len = n - 3; +} + +size_t cst_crlf_to_lf(char *s, size_t n) { + size_t o = 0; + for (size_t i = 0; i < n; i++) + if (!(s[i] == '\r' && i + 1 < n && s[i + 1] == '\n')) + s[o++] = s[i]; + if (o < n) + s[o] = 0; + return o; +} + +int cst_text_is_crlf(const char *s, size_t n) { + int crlf = 0; + for (size_t i = 0; s && i < n; i++) { + if (s[i] != '\n') + continue; + if (i == 0 || s[i - 1] != '\r') + return 0; + crlf = 1; + } + return crlf; +} + +char *cst_text_restyle(const char *s, size_t n, int crlf, int bom, + size_t *out_len) { + size_t skip = cst_bom_len(s, n), extra = bom ? 3 : 0; + for (size_t i = skip; crlf && i < n; i++) + extra += s[i] == '\n' && (i == skip || s[i - 1] != '\r'); + char *o = malloc(n - skip + extra + 1); + if (!o) + return NULL; + size_t k = 0; + if (bom) { + memcpy(o, "\xEF\xBB\xBF", 3); + k = 3; + } + for (size_t i = skip; i < n; i++) { + if (crlf && s[i] == '\n' && (i == skip || s[i - 1] != '\r')) + o[k++] = '\r'; + o[k++] = s[i]; + } + o[k] = 0; + if (out_len) + *out_len = k; + return o; +} + +/* Next code point of a UTF-8 string; a stray byte stands for itself. */ +static int utf8_next(const char **ps) { + const unsigned char *p = (const unsigned char *)*ps; + size_t left = 0; + while (left < 4 && p[left]) + left++; + size_t k = utf8_seq_ok(p, left); + if (k <= 1) { + *ps += 1; + return p[0]; + } + int c = p[0] & (0x7F >> k); + for (size_t i = 1; i < k; i++) + c = (c << 6) | (p[i] & 0x3F); + *ps += k; + return c; +} + +static int fold_cp(int c) { + if (c >= 'A' && c <= 'Z') + return c + 32; + if (c > 0x7F && c <= 0xFFFF) + return (int)towlower((wint_t)c); + return c; +} + +/* p points just past '['. 1/0 = (no) match with *pp moved past ']'; + * -1 = no closing ']' (the '[' is then a literal). */ +static int fnm_bracket(const char **pp, int c, int fold) { + const char *p = *pp; + int neg = 0, hit = 0; + if (*p == '!' || *p == '^') { + neg = 1; + p++; + } + int fc = fold ? fold_cp(c) : c; + for (int first = 1; *p && (*p != ']' || first); first = 0) { + if (p[0] == '[' && p[1] == ':') { + const char *e = strstr(p + 2, ":]"); + if (e) { + char name[16]; + size_t nl = (size_t)(e - (p + 2)); + if (nl < sizeof(name)) { + memcpy(name, p + 2, nl); + name[nl] = 0; + wctype_t t = wctype(name); + if (t && c <= 0xFFFF && iswctype((wint_t)c, t)) + hit = 1; + } + p = e + 2; + continue; + } + } + if (*p == '\\' && p[1]) + p++; + int lo = utf8_next(&p), hi = lo; + if (p[0] == '-' && p[1] && p[1] != ']') { + p++; + if (*p == '\\' && p[1]) + p++; + hi = utf8_next(&p); + } + if ((c >= lo && c <= hi) || + (fold && fc >= fold_cp(lo) && fc <= fold_cp(hi))) + hit = 1; + } + if (*p != ']') + return -1; + *pp = p + 1; + return hit != neg; +} + +int cst_fnmatch(const char *pat, const char *str, int flags) { +#ifdef _WIN32 + flags |= CST_FNM_CASEFOLD; +#endif + int fold = (flags & CST_FNM_CASEFOLD) != 0; + const char *p = pat, *s = str; + const char *star_p = NULL, *star_s = NULL; + if ((flags & CST_FNM_PERIOD) && s[0] == '.' && p[0] != '.' && + !(p[0] == '\\' && p[1] == '.')) + return 1; + while (*s) { + if (*p == '*') { + while (*p == '*') + p++; + if (!*p) + return 0; + star_p = p; + star_s = s; + continue; + } + const char *sn = s; + int c = utf8_next(&sn); + if (*p == '?') { + p++; + s = sn; + continue; + } + if (*p == '[') { + const char *q = p + 1; + int r = fnm_bracket(&q, c, fold); + if (r == 1) { + p = q; + s = sn; + continue; + } + if (r == 0) + goto backtrack; + } + { + const char *q = p; + if (*q == '\\' && q[1]) + q++; + if (*q) { + int pc = utf8_next(&q); + if (pc == c || (fold && fold_cp(pc) == fold_cp(c))) { + p = q; + s = sn; + continue; + } + } + } + backtrack: + if (!star_p) + return 1; + utf8_next(&star_s); + s = star_s; + p = star_p; + } + while (*p == '*') + p++; + return *p ? 1 : 0; +} + uint64_t cst_fnv1a64(uint64_t h, const void *data, size_t n) { const unsigned char *p = data; for (size_t i = 0; i < n; i++) { diff --git a/tests/cst_test_util.h b/tests/cst_test_util.h new file mode 100644 index 0000000..0b36f2e --- /dev/null +++ b/tests/cst_test_util.h @@ -0,0 +1,208 @@ +/* Portable helpers for the test programs: scratch dirs, env, recursive + * delete, symlinks and (with CST_TEST_SOCKETS) a loopback socket layer. */ +#ifndef CST_TEST_UTIL_H +#define CST_TEST_UTIL_H + +#include "os_compat.h" + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef _WIN32 +#ifdef CST_TEST_SOCKETS +#include <winsock2.h> /* before windows.h */ +#include <ws2tcpip.h> +#endif +#include <direct.h> +#include <io.h> +#include <windows.h> +#else +#include <sys/stat.h> +#include <unistd.h> +#endif + +#if defined(__GNUC__) +#define CST_TEST_UNUSED __attribute__((unused)) +#else +#define CST_TEST_UNUSED +#endif + +/* mkdir(path, mode) that also builds on Windows. */ +#define test_mkdir(p, mode) os_mkdir((p), (mode)) + +/* Fresh empty directory "<tmp>/<tag>-<pid>-<n>"; 0 on success. */ +static CST_TEST_UNUSED int test_tmpdir(char *out, size_t n, const char *tag) { + static int seq; + char base[1024]; + if (os_temp_dir(base, sizeof(base)) != 0) + snprintf(base, sizeof(base), "."); + for (int i = 0; i < 100; i++) { + int k = snprintf(out, n, "%s/%s-%d-%d", base, tag, os_getpid(), seq++); + if (k < 0 || (size_t)k >= n) + return -1; +#ifdef _WIN32 + int rc = _mkdir(out); +#else + int rc = mkdir(out, 0755); +#endif + if (rc == 0) + return 0; + if (errno != EEXIST) + return -1; + } + return -1; +} + +/* rm -rf without a shell; links are removed, never followed. */ +static CST_TEST_UNUSED int test_rmrf(const char *path) { + os_stat_t st; + if (os_lstat(path, &st) != 0) + return 0; + if (st.is_link) { +#ifdef _WIN32 + if (RemoveDirectoryA(path) || DeleteFileA(path)) + return 0; + return -1; +#else + return unlink(path); +#endif + } + if (!st.is_dir) { +#ifdef _WIN32 + SetFileAttributesA(path, FILE_ATTRIBUTE_NORMAL); +#endif + return remove(path); + } + os_dir_t *d = os_dir_open(path); + if (d) { + const os_dirent_t *e; + while ((e = os_dir_next(d))) { + if (!strcmp(e->name, ".") || !strcmp(e->name, "..")) + continue; + size_t L = strlen(path) + strlen(e->name) + 2; + char *c = malloc(L); + if (!c) + break; + snprintf(c, L, "%s/%s", path, e->name); + test_rmrf(c); + free(c); + } + os_dir_close(d); + } +#ifdef _WIN32 + return _rmdir(path); +#else + return rmdir(path); +#endif +} + +/* 0 on success; -1 when the platform or account cannot make links (Windows + * without Developer Mode or admin rights): callers skip those checks. */ +static CST_TEST_UNUSED int test_symlink(const char *target, const char *link) { +#ifdef _WIN32 + wchar_t *wt = os_utf8_to_wide(target), *wl = os_utf8_to_wide(link); + int ok = 0; + if (wt && wl) { + for (wchar_t *p = wt; *p; p++) + if (*p == L'/') + *p = L'\\'; + DWORD attr = GetFileAttributesW(wt); + if (attr == INVALID_FILE_ATTRIBUTES) { + /* relative targets resolve against the link's directory */ + wchar_t *slash = wcsrchr(wl, L'/'); + if (!slash) + slash = wcsrchr(wl, L'\\'); + if (slash && wt[0] != L'\\' && !(wt[0] && wt[1] == L':')) { + size_t dl = (size_t)(slash - wl) + 1, tl = wcslen(wt); + wchar_t *full = malloc((dl + tl + 1) * sizeof(wchar_t)); + if (full) { + memcpy(full, wl, dl * sizeof(wchar_t)); + memcpy(full + dl, wt, (tl + 1) * sizeof(wchar_t)); + attr = GetFileAttributesW(full); + free(full); + } + } + } + DWORD flags = 0x2; /* SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE */ + if (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)) + flags |= SYMBOLIC_LINK_FLAG_DIRECTORY; + ok = CreateSymbolicLinkW(wl, wt, flags) != 0; + } + free(wt); + free(wl); + return ok ? 0 : -1; +#else + return symlink(target, link); +#endif +} + +/* rand_r replacement (same sequence everywhere). */ +static CST_TEST_UNUSED int test_rand(unsigned *seed) { + *seed = *seed * 1103515245u + 12345u; + return (int)((*seed >> 16) & 0x7fff); +} + +/* getline replacement: -1 at EOF. */ +static CST_TEST_UNUSED long test_getline(char **line, size_t *cap, FILE *fp) { + if (!*line || *cap == 0) { + *cap = 256; + *line = malloc(*cap); + if (!*line) + return -1; + } + size_t n = 0; + int c; + while ((c = fgetc(fp)) != EOF) { + if (n + 2 > *cap) { + size_t nc = *cap * 2; + char *nl = realloc(*line, nc); + if (!nl) + return -1; + *line = nl; + *cap = nc; + } + (*line)[n++] = (char)c; + if (c == '\n') + break; + } + if (n == 0) + return -1; + (*line)[n] = 0; + return (long)n; +} + +static CST_TEST_UNUSED void test_sleep_ms(int ms) { +#ifdef _WIN32 + Sleep((DWORD)ms); +#else + usleep((useconds_t)ms * 1000); +#endif +} + +#ifdef CST_TEST_SOCKETS +#ifdef _WIN32 +typedef SOCKET test_sock_t; +#define TEST_BAD_SOCK INVALID_SOCKET +#define test_sock_close closesocket +#ifndef MSG_NOSIGNAL +#define MSG_NOSIGNAL 0 +#endif +static CST_TEST_UNUSED void test_sock_init(void) { + WSADATA w; + WSAStartup(MAKEWORD(2, 2), &w); +} +#else +#include <arpa/inet.h> +#include <netinet/in.h> +#include <signal.h> +#include <sys/socket.h> +typedef int test_sock_t; +#define TEST_BAD_SOCK (-1) +#define test_sock_close close +static CST_TEST_UNUSED void test_sock_init(void) { signal(SIGPIPE, SIG_IGN); } +#endif +#endif /* CST_TEST_SOCKETS */ + +#endif diff --git a/tests/test_apply_patch.c b/tests/test_apply_patch.c index 1d39f78..367c5ed 100644 --- a/tests/test_apply_patch.c +++ b/tests/test_apply_patch.c @@ -7,6 +7,7 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" static int nfail; @@ -180,12 +181,12 @@ static void test_tool_multi_file(void) { int main(void) { test_buffer_basic(); test_buffer_edge(); - snprintf(g_ws, sizeof(g_ws), "/tmp/cst-patch-test-%d", (int)getpid()); - mkdir(g_ws, 0755); + if (test_tmpdir(g_ws, sizeof(g_ws), "cst-patch-test") != 0) { + perror("tmpdir"); + return 1; + } test_tool_multi_file(); - char cmd[300]; - snprintf(cmd, sizeof(cmd), "rm -rf %s /tmp/escape.c", g_ws); - if (system(cmd) != 0) + if (test_rmrf(g_ws) != 0) fprintf(stderr, "warning: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_apply_patch: %d failed\n", nfail); diff --git a/tests/test_config.c b/tests/test_config.c index 0b4678d..bcc98d6 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -9,6 +9,11 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" +#ifdef _WIN32 +#include <aclapi.h> +#include <sddl.h> +#endif #define CHECK(c) \ do { \ @@ -23,7 +28,7 @@ static char g_dir[256]; static void write_file(const char *name, const char *text) { char path[512]; snprintf(path, sizeof(path), "%s/cst-agent/%s", g_dir, name); - FILE *fp = fopen(path, "w"); + FILE *fp = fopen(path, "wb"); CHECK(fp); fputs(text, fp); fclose(fp); @@ -57,9 +62,26 @@ static void test_creds(void) { char path[512]; snprintf(path, sizeof(path), "%s/cst-agent/credentials", g_dir); +#ifdef _WIN32 + { + wchar_t *wp = os_utf8_to_wide(path); + PSECURITY_DESCRIPTOR sd = NULL; + LPWSTR s = NULL; + CHECK(wp && GetNamedSecurityInfoW(wp, SE_FILE_OBJECT, + LABEL_SECURITY_INFORMATION, NULL, NULL, + NULL, NULL, &sd) == ERROR_SUCCESS); + CHECK(ConvertSecurityDescriptorToStringSecurityDescriptorW( + sd, SDDL_REVISION_1, LABEL_SECURITY_INFORMATION, &s, NULL)); + CHECK(wcsstr(s, L"NR") && wcsstr(s, L";ME)")); /* Low-IL cannot read */ + LocalFree(s); + LocalFree(sd); + free(wp); + } +#else struct stat st; CHECK(stat(path, &st) == 0); CHECK((st.st_mode & 0077) == 0); +#endif CHECK(cst_creds_resolve("deepseek", key, sizeof(key), &from) == 0); CHECK(strcmp(from, "saved") == 0); @@ -124,23 +146,20 @@ static void test_effort(void) { } int main(void) { - snprintf(g_dir, sizeof(g_dir), "/tmp/cst-config-XXXXXX"); - CHECK(mkdtemp(g_dir)); + CHECK(test_tmpdir(g_dir, sizeof(g_dir), "cst-config") == 0); setenv("XDG_CONFIG_HOME", g_dir, 1); unsetenv("DEEPSEEK_API_KEY"); unsetenv("MINIMAX_API_KEY"); unsetenv("OPENAI_API_KEY"); char sub[300]; snprintf(sub, sizeof(sub), "%s/cst-agent", g_dir); - CHECK(mkdir(sub, 0700) == 0); + CHECK(test_mkdir(sub, 0755) == 0); test_creds(); test_settings(); test_effort(); - char cmd[400]; - snprintf(cmd, sizeof(cmd), "rm -rf '%s'", g_dir); - CHECK(system(cmd) == 0); + CHECK(test_rmrf(g_dir) == 0); printf("test_config: OK\n"); return 0; } diff --git a/tests/test_diff.c b/tests/test_diff.c index 17bc085..178f038 100644 --- a/tests/test_diff.c +++ b/tests/test_diff.c @@ -9,6 +9,7 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include "cst_test_util.h" static int nfail; @@ -30,6 +31,11 @@ static void put(const char *p, const char *s) { /* GNU diff output, or NULL when diff is not installed. */ static char *gnu_diff(const char *a, const char *b) { +#ifdef _WIN32 + (void)a; + (void)b; + return NULL; /* no GNU diff to compare against */ +#else char pa[] = "/tmp/cstdiffaXXXXXX", pb[] = "/tmp/cstdiffbXXXXXX"; int fa = mkstemp(pa), fb = mkstemp(pb); if (fa < 0 || fb < 0) @@ -67,6 +73,7 @@ static char *gnu_diff(const char *a, const char *b) { unlink(pa); unlink(pb); return out; +#endif } static int have_gnu_diff(void) { @@ -130,11 +137,11 @@ static char *random_text(unsigned *seed, int lines) { size_t cap = (size_t)lines * 16 + 2, n = 0; char *s = malloc(cap); for (int i = 0; i < lines; i++) { - const char *w = words[rand_r(seed) % 8]; - n += (size_t)snprintf(s + n, cap - n, "%s%d\n", w, rand_r(seed) % 3); + const char *w = words[test_rand(seed) % 8]; + n += (size_t)snprintf(s + n, cap - n, "%s%d\n", w, test_rand(seed) % 3); } s[n] = 0; - if (n && rand_r(seed) % 5 == 0) + if (n && test_rand(seed) % 5 == 0) s[n - 1] = 0; /* no newline at end of file */ return s; } @@ -146,15 +153,15 @@ static char *mutate(unsigned *seed, const char *a) { while (*p) { const char *e = strchr(p, '\n'); size_t len = e ? (size_t)(e - p) + 1 : strlen(p); - int r = rand_r(seed) % 10; + int r = test_rand(seed) % 10; if (r == 0) { /* drop the line */ } else if (r == 1) { - o += (size_t)snprintf(b + o, cap - o, "inserted%d\n", rand_r(seed) % 9); + o += (size_t)snprintf(b + o, cap - o, "inserted%d\n", test_rand(seed) % 9); memcpy(b + o, p, len); o += len; } else if (r == 2) { - o += (size_t)snprintf(b + o, cap - o, "changed%d\n", rand_r(seed) % 9); + o += (size_t)snprintf(b + o, cap - o, "changed%d\n", test_rand(seed) % 9); } else { memcpy(b + o, p, len); o += len; @@ -198,7 +205,7 @@ int main(void) { unsigned seed = 12345; int ok = 0, total = 400; for (int i = 0; i < total; i++) { - char *a = random_text(&seed, 1 + (int)(rand_r(&seed) % 60)); + char *a = random_text(&seed, 1 + (int)(test_rand(&seed) % 60)); char *b = mutate(&seed, a); ok += round_trip(a, b); free(a); diff --git a/tests/test_lowil.c b/tests/test_lowil.c new file mode 100644 index 0000000..0b0d6ea --- /dev/null +++ b/tests/test_lowil.c @@ -0,0 +1,184 @@ +/* Windows Low-IL shell sandbox: writes only in the workspace and the + * sandbox TEMP, the credentials file is unreadable, timeouts end the whole + * process tree, and /sandbox reset-labels restores the workspace label. + * Elsewhere it only checks that the kind is never reported. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "cst_sandbox.h" +#include "os_compat.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "cst_test_util.h" + +static int nfail; + +#define EXPECT(cond, msg) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ + nfail++; \ + } \ + } while (0) + +#ifdef _WIN32 +static int run(const char *ws, const char *cmd, os_run_result_t *r) { + memset(r, 0, sizeof(*r)); + if (cst_sandbox_run_shell(ws, cmd, 0, r) != 0) + return -1; + return r->exit_code; +} + +static int has(const os_run_result_t *r, const char *s) { + return r->out && strstr(r->out, s); +} + +static void write_file(const char *path, const char *text) { + FILE *f = fopen(path, "wb"); + if (f) { + fputs(text, f); + fclose(f); + } +} + +static void confined(void) { + char root[1024], cfg[1100], ws[1100], out[1100], cred[1300], cmd[4096]; + char wsr[1100]; + if (test_tmpdir(root, sizeof(root), "cst-lowil") != 0) { + EXPECT(0, "tmpdir"); + return; + } + snprintf(cfg, sizeof(cfg), "%s/cfg", root); + snprintf(ws, sizeof(ws), "%s/项目 ws", root); + snprintf(out, sizeof(out), "%s/outside", root); + test_mkdir(cfg, 0700); + test_mkdir(ws, 0755); + test_mkdir(out, 0755); + setenv("XDG_CONFIG_HOME", cfg, 1); + char cdir[1200]; + EXPECT(os_config_dir(cdir, sizeof(cdir)) == 0, "config dir"); + os_mkdir_p(cdir, 0700); + snprintf(cred, sizeof(cred), "%s/credentials", cdir); + write_file(cred, "deepseek=sk-test-secret\n"); + EXPECT(os_chmod_private(cred) == 0, "label credentials"); + EXPECT(os_realpath(ws, wsr, sizeof(wsr)) == 0, "realpath ws"); + os_path_norm_slashes(wsr); + + os_run_result_t r; + EXPECT(run(wsr, "echo hi > in.txt && cat in.txt", &r) == 0, + "write inside the workspace"); + EXPECT(has(&r, "hi"), "output captured"); + EXPECT(has(&r, "labeled"), "first run says the workspace was labeled"); + os_run_result_free(&r); + + EXPECT(run(wsr, "echo again > in.txt && cat in.txt", &r) == 0, + "second run"); + EXPECT(!has(&r, "labeled"), "labeled only once"); + EXPECT(!has(&r, "background"), "conhost is not a background process"); + os_run_result_free(&r); + + char *labels = cst_sandbox_labels(); + EXPECT(labels && strstr(labels, wsr), "workspace recorded"); + free(labels); + + snprintf(cmd, sizeof(cmd), "echo x > '%s/o.txt'", out); + int rc = run(wsr, cmd, &r); + EXPECT(rc != 0, "write outside the workspace fails"); + EXPECT(cst_sandbox_blocked(r.out, r.out_len, rc), "outside write blocked"); + os_run_result_free(&r); + snprintf(cmd, sizeof(cmd), "%s/o.txt", out); + os_stat_t st; + EXPECT(os_lstat(cmd, &st) != 0, "nothing written outside"); + + snprintf(cmd, sizeof(cmd), "cat '%s'", cred); + rc = run(wsr, cmd, &r); + EXPECT(rc != 0 && !has(&r, "sk-test-secret"), "credentials unreadable"); + EXPECT(cst_sandbox_blocked(r.out, r.out_len, rc), "credential read blocked"); + os_run_result_free(&r); + + EXPECT(run(wsr, "echo t > \"$TEMP/t.txt\" && cat \"$TEMP/t.txt\"", &r) == 0, + "TEMP is writable"); + EXPECT(has(&r, "t"), "TEMP read back"); + os_run_result_free(&r); + + /* System32's whoami: busybox has an applet and MSYS2's cannot start at + * Low IL */ + char sys[MAX_PATH]; + UINT sl = GetSystemDirectoryA(sys, sizeof(sys)); + for (UINT i = 0; i < sl; i++) + if (sys[i] == '\\') + sys[i] = '/'; + snprintf(cmd, sizeof(cmd), "'%s/whoami.exe' /groups | grep S-1-16-4096", + sys); + rc = run(wsr, cmd, &r); + EXPECT(rc == 0, "child runs at Low integrity"); + if (rc != 0) + fprintf(stderr, "whoami: exit %d: %.*s\n", rc, (int)r.out_len, + r.out ? r.out : ""); + os_run_result_free(&r); + + /* timeout ends background children too */ + os_set_run_timeout(2); + double t0 = os_now_ms(); + rc = run(wsr, "sleep 30 & sleep 30; echo late", &r); + double ms = os_now_ms() - t0; + EXPECT(rc == 124, "timeout exit code"); + EXPECT(ms < 10000, "process tree killed promptly"); + EXPECT(!has(&r, "late"), "command did not finish"); + os_run_result_free(&r); + os_set_run_timeout(0); + + const char *prof = getenv("USERPROFILE"); + char profr[1100]; + if (prof && os_realpath(prof, profr, sizeof(profr)) == 0) { + os_path_norm_slashes(profr); + rc = run(profr, "echo no", &r); + EXPECT(rc == 126 && has(&r, "will not label"), "profile is not labeled"); + EXPECT(cst_sandbox_blocked(r.out, r.out_len, rc), "offers a rerun"); + os_run_result_free(&r); + } + rc = run("C:/", "echo no", &r); + EXPECT(rc == 126 && has(&r, "will not label"), "drive root is not labeled"); + os_run_result_free(&r); + + char *rep = NULL; + EXPECT(cst_sandbox_reset_labels(&rep) >= 1, "reset-labels restored"); + EXPECT(rep && strstr(rep, "restored"), "reset report"); + free(rep); + labels = cst_sandbox_labels(); + EXPECT(!labels, "record cleared"); + free(labels); + char sddl[256]; + EXPECT(os_win_get_label(wsr, sddl, sizeof(sddl)) == 0 && + strcmp(sddl, "S:") == 0, + "workspace has no explicit label again"); + + unsetenv("XDG_CONFIG_HOME"); + test_rmrf(root); +} +#endif + +int main(void) { + int kind = cst_sandbox_probe(); + printf("test_lowil: sandbox %s\n", cst_sandbox_name(kind)); + EXPECT(strcmp(cst_sandbox_name(CST_SANDBOX_LOWIL), "lowil") == 0, "name"); +#ifdef _WIN32 + EXPECT(kind == CST_SANDBOX_LOWIL, "lowil available"); + if (kind == CST_SANDBOX_LOWIL) + confined(); +#else + EXPECT(kind != CST_SANDBOX_LOWIL, "lowil is Windows only"); + char *rep = NULL; + EXPECT(cst_sandbox_reset_labels(&rep) == 0 && !rep, "nothing to reset"); + EXPECT(!cst_sandbox_labels(), "no labels"); +#endif + if (nfail) { + fprintf(stderr, "test_lowil: %d failure(s)\n", nfail); + return 1; + } + puts("test_lowil: OK"); + return 0; +} diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6ee86db..bf8c50e 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -14,7 +14,10 @@ #include <string.h> #include <unistd.h> +#include "cst_test_util.h" + static int nfail; +static char g_tmp[OS_PATH_MAX]; #define CHECK(cond, msg) \ do { \ @@ -159,7 +162,7 @@ static int call(const char *tool, const char *k, const char *v, } cst_tool_ctx_t ctx; memset(&ctx, 0, sizeof(ctx)); - ctx.workspace_root = "/tmp"; + ctx.workspace_root = g_tmp; memset(r, 0, sizeof(*r)); return cst_tool_run(&ctx, &c, r); } @@ -181,6 +184,8 @@ int main(int argc, char **argv) { fprintf(stderr, "cannot resolve %s\n", argv[0]); return 1; } + if (os_temp_dir(g_tmp, sizeof(g_tmp)) != 0) + snprintf(g_tmp, sizeof(g_tmp), "."); setenv("CST_TEST_SECRET", "s3cret", 1); char cfg[OS_PATH_MAX + 128]; snprintf(cfg, sizeof(cfg), @@ -206,7 +211,7 @@ int main(int argc, char **argv) { cst_mcp_counts(&ready, &total); CHECK(ready == 0 && total == 3, "counts before start (disabled excluded)"); - cst_mcp_start_all("/tmp"); + cst_mcp_start_all(g_tmp); cst_mcp_wait_ready(10000); cst_mcp_info_t mi[CST_MCP_MAX]; @@ -281,7 +286,7 @@ int main(int argc, char **argv) { /* a live server stops cleanly too */ CHECK(cst_mcp_add_config_json("fake2", cfg, NULL, "user") == 0, "re-add"); - cst_mcp_start_all("/tmp"); + cst_mcp_start_all(g_tmp); cst_mcp_wait_ready(10000); CHECK(cst_tool_find("mcp__fake2__echo"), "second server ready"); t0 = os_now_ms(); diff --git a/tests/test_models.c b/tests/test_models.c index 11246d6..00c368c 100644 --- a/tests/test_models.c +++ b/tests/test_models.c @@ -8,18 +8,10 @@ #include <stdlib.h> #include <string.h> -#ifdef _WIN32 -int main(void) { - puts("test_models: skipped on Windows"); - return 0; -} -#else +#define CST_TEST_SOCKETS +#include "cst_test_util.h" -#include <arpa/inet.h> -#include <netinet/in.h> #include <pthread.h> -#include <sys/socket.h> -#include <unistd.h> #define CHECK(c) \ do { \ @@ -32,35 +24,38 @@ int main(void) { /* One canned HTTP response per accepted connection. */ static const char *g_responses[8]; static int g_nresp; -static int g_listen = -1; +static test_sock_t g_listen = TEST_BAD_SOCK; static char g_request[4096]; static void *serve(void *arg) { (void)arg; for (int i = 0; i < g_nresp; i++) { - int c = accept(g_listen, NULL, NULL); - if (c < 0) + test_sock_t c = accept(g_listen, NULL, NULL); + if (c == TEST_BAD_SOCK) return NULL; - ssize_t n = recv(c, g_request, sizeof(g_request) - 1, 0); + long n = recv(c, g_request, sizeof(g_request) - 1, 0); g_request[n > 0 ? n : 0] = 0; const char *r = g_responses[i]; - ssize_t off = 0, len = (ssize_t)strlen(r); + long off = 0, len = (long)strlen(r); while (off < len) { - ssize_t w = send(c, r + off, (size_t)(len - off), 0); + long w = send(c, r + off, (int)(len - off), 0); if (w <= 0) break; off += w; } - close(c); + test_sock_close(c); } return NULL; } static int start_server(void) { + test_sock_init(); g_listen = socket(AF_INET, SOCK_STREAM, 0); - CHECK(g_listen >= 0); + CHECK(g_listen != TEST_BAD_SOCK); +#ifndef _WIN32 int one = 1; setsockopt(g_listen, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#endif struct sockaddr_in a; memset(&a, 0, sizeof(a)); a.sin_family = AF_INET; @@ -118,7 +113,7 @@ int main(void) { CHECK(n == -1 && strstr(err, "500") != NULL); pthread_join(th, NULL); - close(g_listen); + test_sock_close(g_listen); /* nothing listening: network error, not an auth failure */ n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err)); @@ -127,4 +122,3 @@ int main(void) { puts("test_models: ok"); return 0; } -#endif diff --git a/tests/test_os_dir.c b/tests/test_os_dir.c new file mode 100644 index 0000000..1e1156d --- /dev/null +++ b/tests/test_os_dir.c @@ -0,0 +1,147 @@ +/* cst_fnmatch and the os_dir / os_lstat layer that replaced the POSIX-only + * directory code; runs on every platform. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "cst_util.h" +#include "os_compat.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "cst_test_util.h" + +static int nfail; + +#define EXPECT(cond, msg) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ + nfail++; \ + } \ + } while (0) + +static void fnm(const char *pat, const char *s, int flags, int want) { + int got = cst_fnmatch(pat, s, flags) == 0; + if (got != want) { + fprintf(stderr, "FAIL fnmatch(\"%s\", \"%s\", %d) = %d\n", pat, s, flags, + got); + nfail++; + } +} + +static void test_fnmatch(void) { + fnm("*.c", "main.c", 0, 1); + fnm("*.c", "main.h", 0, 0); + fnm("*.c", "src/main.c", 0, 1); /* '*' crosses '/' */ + fnm("?.txt", "a.txt", 0, 1); + fnm("?.txt", "ab.txt", 0, 0); + fnm("?.txt", "中.txt", 0, 1); /* one code point */ + fnm("[abc]x", "bx", 0, 1); + fnm("[!abc]x", "bx", 0, 0); + fnm("[a-c]x", "cx", 0, 1); + fnm("[[:digit:]]*", "7up", 0, 1); + fnm("[[:digit:]]*", "up", 0, 0); + fnm("\\*", "*", 0, 1); + fnm("\\*", "a", 0, 0); + fnm("*", ".hidden", CST_FNM_PERIOD, 0); + fnm(".*", ".hidden", CST_FNM_PERIOD, 1); + fnm("*", ".hidden", 0, 1); + fnm("*文件*", "我的文件.txt", 0, 1); + fnm("README*", "readme.md", CST_FNM_CASEFOLD, 1); +#ifdef _WIN32 + fnm("README*", "readme.md", 0, 1); /* Windows names are case-insensitive */ +#else + fnm("README*", "readme.md", 0, 0); +#endif + fnm("", "", 0, 1); + fnm("a", "", 0, 0); +} + +static void touch(const char *dir, const char *name) { + char p[1200]; + snprintf(p, sizeof(p), "%s/%s", dir, name); + FILE *f = fopen(p, "wb"); + if (f) { + fputs("x", f); + fclose(f); + } +} + +static void test_dir(void) { + char root[1024], sub[1100], p[1200]; + if (test_tmpdir(root, sizeof(root), "cst-osdir") != 0) { + EXPECT(0, "tmpdir"); + return; + } + snprintf(sub, sizeof(sub), "%s/子目录", root); + EXPECT(test_mkdir(sub, 0755) == 0, "mkdir with a Chinese name"); + touch(root, "a.txt"); + touch(root, "文件 一.md"); + touch(root, ".hidden"); + + int files = 0, dirs = 0, seen_cn = 0, seen_hidden = 0, dots = 0; + os_dir_t *d = os_dir_open(root); + EXPECT(d != NULL, "os_dir_open"); + const os_dirent_t *e; + while (d && (e = os_dir_next(d)) != NULL) { + if (!strcmp(e->name, ".") || !strcmp(e->name, "..")) + dots++; + if (e->is_dir) + dirs++; + else + files++; + seen_cn |= strcmp(e->name, "文件 一.md") == 0 && !e->is_dir; + seen_hidden |= strcmp(e->name, ".hidden") == 0; + if (!strcmp(e->name, "子目录")) + EXPECT(e->is_dir && !e->is_link, "subdir flags"); + } + os_dir_close(d); + EXPECT(dots == 0, "no . or .. entries"); + EXPECT(files == 3 && dirs == 1, "entry counts"); + EXPECT(seen_cn, "UTF-8 file name listed"); + EXPECT(seen_hidden, "dot files listed"); + d = os_dir_open(sub); + EXPECT(d != NULL && os_dir_next(d) == NULL, "empty Chinese subdir"); + if (d) + os_dir_close(d); + + os_stat_t st; + snprintf(p, sizeof(p), "%s/文件 一.md", root); + EXPECT(os_lstat(p, &st) == 0 && st.is_reg && !st.is_dir && st.size == 1, + "os_lstat on a UTF-8 file"); + EXPECT(os_lstat(sub, &st) == 0 && st.is_dir, "os_lstat on a dir"); + snprintf(p, sizeof(p), "%s/missing", root); + EXPECT(os_lstat(p, &st) != 0, "os_lstat on a missing path"); + EXPECT(os_dir_open(p) == NULL, "os_dir_open on a missing path"); + + char real[1200], rr[1200]; + EXPECT(os_realpath(sub, real, sizeof(real)) == 0 && + os_realpath(root, rr, sizeof(rr)) == 0, + "os_realpath"); + EXPECT(os_path_under(rr, real), "realpath of subdir is under root"); + EXPECT(!os_path_under(real, rr), "root is not under subdir"); + + snprintf(p, sizeof(p), "%s/link", root); + int lk = test_symlink(sub, p); + if (lk == 0) { + EXPECT(os_lstat(p, &st) == 0 && st.is_link, "symlink is reported as link"); + } else { + printf("test_os_dir: symlinks not permitted, link checks skipped\n"); + } + test_rmrf(root); + EXPECT(os_lstat(root, &st) != 0, "test_rmrf removed everything"); +} + +int main(void) { + os_console_init(); + test_fnmatch(); + test_dir(); + if (nfail) { + fprintf(stderr, "test_os_dir: %d failure(s)\n", nfail); + return 1; + } + puts("test_os_dir: OK"); + return 0; +} diff --git a/tests/test_plugins.c b/tests/test_plugins.c index 93bd168..743f6ba 100644 --- a/tests/test_plugins.c +++ b/tests/test_plugins.c @@ -14,6 +14,7 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" static int nfail; @@ -33,11 +34,11 @@ static void put(const char *rel, const char *text) { for (char *q = p + strlen(g_root) + 1; *q; q++) { if (*q == '/') { *q = 0; - mkdir(p, 0755); + test_mkdir(p, 0755); *q = '/'; } } - FILE *fp = fopen(p, "w"); + FILE *fp = fopen(p, "wb"); if (fp) { fputs(text, fp); fclose(fp); @@ -70,8 +71,10 @@ static void run(const char *ws, const char *tool, const char *k1, } int main(void) { - snprintf(g_root, sizeof(g_root), "/tmp/cst-plugins-%d", (int)getpid()); - mkdir(g_root, 0755); + if (test_tmpdir(g_root, sizeof(g_root), "cst-plugins") != 0) { + perror("tmpdir"); + return 1; + } char ws[300], cfgdir[300]; snprintf(ws, sizeof(ws), "%s/ws", g_root); snprintf(cfgdir, sizeof(cfgdir), "%s/cfg", g_root); @@ -206,9 +209,7 @@ int main(void) { CHECK(!cst_tool_find("skill") && !cst_tool_find("echo_args"), "extensions unload"); - char cmd[400]; - snprintf(cmd, sizeof(cmd), "rm -rf %s", g_root); - if (system(cmd) != 0) + if (test_rmrf(g_root) != 0) fprintf(stderr, "note: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_plugins: %d failure(s)\n", nfail); diff --git a/tests/test_replay_real.c b/tests/test_replay_real.c index 4eeb9ec..7a4a694 100644 --- a/tests/test_replay_real.c +++ b/tests/test_replay_real.c @@ -8,6 +8,7 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include "cst_test_util.h" /* * Truncation experiment on recorded model streams (tests/fixtures/sse). @@ -79,7 +80,7 @@ static int load_trace(const char *path, trace_t *t) { char *line = NULL; size_t cap = 0; ssize_t n; - while ((n = getline(&line, &cap, fp)) > 0) { + while ((n = test_getline(&line, &cap, fp)) > 0) { if (strncmp(line, "data:", 5) != 0) continue; cJSON *j = cJSON_Parse(line + 5); diff --git a/tests/test_sandbox.c b/tests/test_sandbox.c index c61130f..3375982 100644 --- a/tests/test_sandbox.c +++ b/tests/test_sandbox.c @@ -73,6 +73,43 @@ static void test_blocked(void) { const char *d = "curl: (7) Failed to connect to example.com port 443 after 9 " "ms: Could not connect to server\n"; CHECK(cst_sandbox_blocked(d, strlen(d), 7)); + const char *e = "Access is denied.\r\n"; + CHECK(cst_sandbox_blocked(e, strlen(e), 1)); + const char *f = "拒绝访问。\r\n"; + CHECK(cst_sandbox_blocked(f, strlen(f), 1)); + const char *g = "sh: C:/msys64/usr/bin/ls.exe: A dynamic link library (DLL) " + "initialization routine failed. Error 0xc0000142\n"; + CHECK(cst_sandbox_blocked(g, strlen(g), 1)); +} + +static void test_network_rules(void) { + static const char *const NET[] = { + "curl -fsSL https://example.com", + "cd x && wget http://a/b", + "/usr/bin/ssh host ls", + "C:\\tools\\curl.exe -O x", + "git push origin main", + "git --no-pager fetch", + "npm install", + "python -m pip install requests", + "go get example.com/x", + "cargo install ripgrep", + "Invoke-WebRequest -Uri x", + }; + static const char *const LOCAL[] = { + "ls -la", "git status", "git commit -m push", "npm run build", + "make -j8", "pip list", "echo curly", "cat curl.txt", + }; + for (size_t i = 0; i < sizeof(NET) / sizeof(NET[0]); i++) { + if (!cst_shell_uses_network(NET[i])) + fprintf(stderr, "expected network: %s\n", NET[i]); + CHECK(cst_shell_uses_network(NET[i])); + } + for (size_t i = 0; i < sizeof(LOCAL) / sizeof(LOCAL[0]); i++) { + if (cst_shell_uses_network(LOCAL[i])) + fprintf(stderr, "expected local: %s\n", LOCAL[i]); + CHECK(!cst_shell_uses_network(LOCAL[i])); + } } #if !defined(_WIN32) @@ -314,6 +351,7 @@ static void test_confined(int kind) { int main(void) { test_rules(); test_blocked(); + test_network_rules(); int kind = cst_sandbox_probe(); printf("test_sandbox: sandbox %s\n", cst_sandbox_name(kind)); #if !defined(_WIN32) diff --git a/tests/test_session_roundtrip.c b/tests/test_session_roundtrip.c index b6e627d..a92be93 100644 --- a/tests/test_session_roundtrip.c +++ b/tests/test_session_roundtrip.c @@ -2,6 +2,7 @@ #include "cJSON.h" #include "cst_agent.h" +#include "os_compat.h" #include <stdio.h> #include <stdlib.h> @@ -76,8 +77,11 @@ static void check_messages(cst_session_t *s) { } int main(void) { - char path[256]; - snprintf(path, sizeof(path), "/tmp/cst-session-test-%d.jsonl", (int)getpid()); + char path[1024], tmp[900]; + if (os_temp_dir(tmp, sizeof(tmp)) != 0) + snprintf(tmp, sizeof(tmp), "."); + snprintf(path, sizeof(path), "%s/cst-session-test-%d.jsonl", tmp, + os_getpid()); remove(path); cst_session_t *s = cst_session_open(path); @@ -129,7 +133,7 @@ int main(void) { cst_session_close(s); /* legacy format: tool_calls_json stored as a string */ - fp = fopen(path, "w"); + fp = fopen(path, "wb"); fprintf(fp, "{\"role\":\"assistant\",\"content\":\"\",\"tool_calls_json\":" "\"[{\\\"id\\\":\\\"c9\\\",\\\"type\\\":\\\"function\\\"," "\\\"function\\\":{\\\"name\\\":\\\"run_shell\\\"," diff --git a/tests/test_shell.c b/tests/test_shell.c index f42601b..6cf6188 100644 --- a/tests/test_shell.c +++ b/tests/test_shell.c @@ -8,6 +8,7 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" /* run_shell must never hang the agent: timeout kills the whole process * group, stdin is empty, huge output is capped. */ @@ -41,8 +42,10 @@ static int sh(const char *cmd, cst_tool_result_t *r, double *ms) { } int main(void) { - snprintf(g_ws, sizeof(g_ws), "/tmp/cst-shell-test-%d", (int)getpid()); - mkdir(g_ws, 0755); + if (test_tmpdir(g_ws, sizeof(g_ws), "cst-shell-test") != 0) { + perror("tmpdir"); + return 1; + } cst_tool_result_t r; double ms; @@ -81,9 +84,7 @@ int main(void) { EXPECT(ms < 20000, "large output does not hang"); cst_tool_result_free(&r); - char cmd[300]; - snprintf(cmd, sizeof(cmd), "rm -rf %s", g_ws); - if (system(cmd) != 0) + if (test_rmrf(g_ws) != 0) fprintf(stderr, "warning: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_shell: %d failed\n", nfail); diff --git a/tests/test_skills.c b/tests/test_skills.c index 1cde3fc..c9d8943 100644 --- a/tests/test_skills.c +++ b/tests/test_skills.c @@ -11,6 +11,7 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" static int nfail; @@ -30,11 +31,11 @@ static void put(const char *rel, const char *text) { for (char *q = p + strlen(g_root) + 1; *q; q++) { if (*q == '/') { *q = 0; - mkdir(p, 0755); + test_mkdir(p, 0755); *q = '/'; } } - FILE *fp = fopen(p, "w"); + FILE *fp = fopen(p, "wb"); if (fp) { fputs(text, fp); fclose(fp); @@ -69,8 +70,10 @@ static void frontmatter(void) { int main(void) { frontmatter(); - snprintf(g_root, sizeof(g_root), "/tmp/cst-skills-%d", (int)getpid()); - mkdir(g_root, 0755); + if (test_tmpdir(g_root, sizeof(g_root), "cst-skills") != 0) { + perror("tmpdir"); + return 1; + } char ws[300], cfgdir[300]; snprintf(ws, sizeof(ws), "%s/ws", g_root); snprintf(cfgdir, sizeof(cfgdir), "%s/cfg", g_root); @@ -94,8 +97,8 @@ int main(void) { char link[512], target[512]; snprintf(link, sizeof(link), "%s/ws/.agent/skills/forms/leak", g_root); snprintf(target, sizeof(target), "%s/ws/secret.txt", g_root); - if (symlink(target, link) != 0) - perror("symlink"); + if (test_symlink(target, link) != 0) + fprintf(stderr, "note: no symlinks here, escape check is partial\n"); } cst_skill_t v[CST_SKILLS_MAX]; @@ -180,9 +183,7 @@ int main(void) { CHECK(sp && !strstr(sp, "Skills ("), "minimal prompt unchanged"); free(sp); - char cmd[400]; - snprintf(cmd, sizeof(cmd), "rm -rf %s", g_root); - if (system(cmd) != 0) + if (test_rmrf(g_root) != 0) fprintf(stderr, "note: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_skills: %d failure(s)\n", nfail); diff --git a/tests/test_sse_decode.c b/tests/test_sse_decode.c index e772451..81a3b4a 100644 --- a/tests/test_sse_decode.c +++ b/tests/test_sse_decode.c @@ -7,6 +7,7 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include "cst_test_util.h" /* Drives cst_llm_chat_stream through CST_REPLAY_SSE with synthetic and * recorded (tests/fixtures/sse) streams, at several network chunk sizes. */ @@ -279,18 +280,14 @@ static void test_recorded(void) { } int main(void) { - snprintf(g_dir, sizeof(g_dir), "/tmp/cst-sse-test-%d", (int)getpid()); - char cmd[300]; - snprintf(cmd, sizeof(cmd), "mkdir -p %s", g_dir); - if (system(cmd) != 0) + if (test_tmpdir(g_dir, sizeof(g_dir), "cst-sse-test") != 0) return 1; test_text_reasoning_usage(); test_parallel_calls(); test_same_index_new_id(); test_invalid_args_and_error(); test_recorded(); - snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dir); - if (system(cmd) != 0) + if (test_rmrf(g_dir) != 0) fprintf(stderr, "warning: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_sse_decode: %d failed\n", nfail); diff --git a/tests/test_subagent.c b/tests/test_subagent.c index 2f9d96b..21e2bc8 100644 --- a/tests/test_subagent.c +++ b/tests/test_subagent.c @@ -15,19 +15,16 @@ #include "cst_util.h" #include "os_compat.h" -#include <arpa/inet.h> +#define CST_TEST_SOCKETS +#include "cst_test_util.h" + #include <curl/curl.h> -#include <netinet/in.h> #include <pthread.h> -#include <signal.h> #include <stdatomic.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <sys/socket.h> -#include <sys/stat.h> -#include <unistd.h> static int nfail; @@ -45,9 +42,9 @@ static atomic_int g_child_active, g_child_max, g_child_reqs; static atomic_int g_child_saw_task, g_child_saw_write, g_child_role_prompt; static atomic_int g_parent_saw_task; -static void send_all(int fd, const char *s, size_t n) { +static void send_all(test_sock_t fd, const char *s, size_t n) { while (n > 0) { - ssize_t w = send(fd, s, n, MSG_NOSIGNAL); + long w = send(fd, s, (int)n, MSG_NOSIGNAL); if (w <= 0) return; s += w; @@ -55,7 +52,7 @@ static void send_all(int fd, const char *s, size_t n) { } } -static void sse(int fd, cJSON *delta, const char *finish) { +static void sse(test_sock_t fd, cJSON *delta, const char *finish) { cJSON *o = cJSON_CreateObject(); cJSON *ch = cJSON_AddArrayToObject(o, "choices"); cJSON *c0 = cJSON_CreateObject(); @@ -80,7 +77,7 @@ static void sse(int fd, cJSON *delta, const char *finish) { cJSON_Delete(o); } -static void reply_text(int fd, const char *text) { +static void reply_text(test_sock_t fd, const char *text) { cJSON *d = cJSON_CreateObject(); cJSON_AddStringToObject(d, "role", "assistant"); cJSON_AddStringToObject(d, "content", text); @@ -90,7 +87,7 @@ static void reply_text(int fd, const char *text) { } /* calls: {name, args_json} pairs */ -static void reply_calls(int fd, const char *const *calls, int n) { +static void reply_calls(test_sock_t fd, const char *const *calls, int n) { for (int i = 0; i < n; i++) { cJSON *d = cJSON_CreateObject(); cJSON *tc = cJSON_AddArrayToObject(d, "tool_calls"); @@ -121,7 +118,7 @@ static int has_tool(const cJSON *body, const char *name) { return 0; } -static void handle(int fd, const char *body_txt) { +static void handle(test_sock_t fd, const char *body_txt) { cJSON *body = cJSON_Parse(body_txt); const cJSON *msgs = cJSON_GetObjectItemCaseSensitive(body, "messages"); int nm = cJSON_GetArraySize(msgs); @@ -155,9 +152,9 @@ static void handle(int fd, const char *body_txt) { if (strstr(sys->valuestring, "Role instructions:\nYou only read.")) atomic_store(&g_child_role_prompt, 1); if (strstr(user, "Slow")) { - usleep(4000 * 1000); /* cancelled before this ends */ + test_sleep_ms(4000); /* cancelled before this ends */ } else { - usleep(300 * 1000); + test_sleep_ms(300); } if (strcmp(last_role, "user") == 0) { const char *calls[] = {"read_file", "{\"path\":\"a.txt\"}"}; @@ -206,13 +203,13 @@ static void handle(int fd, const char *body_txt) { } static void *conn_main(void *p) { - int fd = (int)(intptr_t)p; + test_sock_t fd = (test_sock_t)(intptr_t)p; cst_buf_t req = {0}; char tmp[8192]; long need = -1; size_t hdr_end = 0; for (;;) { - ssize_t n = recv(fd, tmp, sizeof(tmp), 0); + long n = recv(fd, tmp, sizeof(tmp), 0); if (n <= 0) break; cst_buf_append(&req, tmp, (size_t)n); @@ -230,23 +227,23 @@ static void *conn_main(void *p) { if (hdr_end) handle(fd, req.buf + hdr_end); cst_buf_free(&req); - close(fd); + test_sock_close(fd); return NULL; } -static int g_lfd; +static test_sock_t g_lfd; static void *server_main(void *p) { (void)p; for (;;) { - int fd = accept(g_lfd, NULL, NULL); - if (fd < 0) + test_sock_t fd = accept(g_lfd, NULL, NULL); + if (fd == TEST_BAD_SOCK) return NULL; pthread_t th; if (pthread_create(&th, NULL, conn_main, (void *)(intptr_t)fd) == 0) pthread_detach(th); else - close(fd); + test_sock_close(fd); } } @@ -282,10 +279,10 @@ static void put(const char *root, const char *rel, const char *text) { for (char *q = p + strlen(root) + 1; *q; q++) if (*q == '/') { *q = 0; - mkdir(p, 0755); + test_mkdir(p, 0755); *q = '/'; } - FILE *fp = fopen(p, "w"); + FILE *fp = fopen(p, "wb"); if (fp) { fputs(text, fp); fclose(fp); @@ -293,13 +290,13 @@ static void put(const char *root, const char *rel, const char *text) { } static void *cancel_later(void *p) { - usleep(800 * 1000); + test_sleep_ms(800); cst_agent_request_cancel(p); return NULL; } int main(void) { - signal(SIGPIPE, SIG_IGN); + test_sock_init(); curl_global_init(CURL_GLOBAL_DEFAULT); g_lfd = socket(AF_INET, SOCK_STREAM, 0); struct sockaddr_in addr; @@ -316,10 +313,10 @@ int main(void) { pthread_t srv; pthread_create(&srv, NULL, server_main, NULL); - char root[256], ws[300]; - snprintf(root, sizeof(root), "/tmp/cst-sub-%d", (int)getpid()); + char root[600], ws[700]; + if (test_tmpdir(root, sizeof(root), "cst-sub") != 0) + return 1; snprintf(ws, sizeof(ws), "%s/ws", root); - mkdir(root, 0755); setenv("XDG_CONFIG_HOME", root, 1); put(root, "ws/a.txt", "alpha\n"); put(root, "ws/.agent/agents/reviewer.md", @@ -470,11 +467,9 @@ int main(void) { CHECK(cst_subagent_list(ws, "parent1", recs, 8) == 0, "delete removes sub-agent transcripts"); - char cmd[400]; - snprintf(cmd, sizeof(cmd), "rm -rf %s", root); - if (system(cmd) != 0) + if (test_rmrf(root) != 0) fprintf(stderr, "note: cleanup failed\n"); - close(g_lfd); + test_sock_close(g_lfd); if (nfail) { fprintf(stderr, "test_subagent: %d failure(s)\n", nfail); return 1; diff --git a/tests/test_tools_fs.c b/tests/test_tools_fs.c index 3c02949..906e307 100644 --- a/tests/test_tools_fs.c +++ b/tests/test_tools_fs.c @@ -12,6 +12,8 @@ #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" + static int nfail; #define EXPECT(cond, msg) \ @@ -98,17 +100,19 @@ static int run(cst_tool_ctx_t *ctx, ctl_tool_call_t c, cst_tool_result_t *r) { static void reset(cst_tool_result_t *r) { cst_tool_result_free(r); } int main(void) { - snprintf(g_ws, sizeof(g_ws), "/tmp/cst-tools-test-%d", (int)getpid()); - mkdir(g_ws, 0755); + if (test_tmpdir(g_ws, sizeof(g_ws), "cst-tools-test") != 0) { + perror("tmpdir"); + return 1; + } char p[512]; snprintf(p, sizeof(p), "%s/src", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); snprintf(p, sizeof(p), "%s/src/deep", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); snprintf(p, sizeof(p), "%s/node_modules", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); snprintf(p, sizeof(p), "%s/build", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); wfile("src/main.c", "#include <stdio.h>\nint main(void) {\n int x = 42;\n" " printf(\"%d\\n\", x);\n return 0;\n}\n"); wfile("src/deep/util.c", "int add(int a, int b) { return a + b; }\n" @@ -119,7 +123,7 @@ int main(void) { wfile(".gitignore", "build/\n*.log\n"); wfile("run.log", "int add in log\n"); { - FILE *fp = fopen(strcat(strcpy(p, g_ws), "/lines.txt"), "w"); + FILE *fp = fopen(strcat(strcpy(p, g_ws), "/lines.txt"), "wb"); for (int i = 1; i <= 100; i++) fprintf(fp, "line %d\n", i); fclose(fp); @@ -410,9 +414,7 @@ int main(void) { "patch targets"); } - char cmd[400]; - snprintf(cmd, sizeof(cmd), "rm -rf %s", g_ws); - if (system(cmd) != 0) + if (test_rmrf(g_ws) != 0) fprintf(stderr, "note: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_tools_fs: %d failure(s)\n", nfail); diff --git a/tests/test_undo.c b/tests/test_undo.c index 3d7b6f2..5ef0e59 100644 --- a/tests/test_undo.c +++ b/tests/test_undo.c @@ -10,6 +10,7 @@ #include <string.h> #include <sys/stat.h> #include <unistd.h> +#include "cst_test_util.h" static int nfail; @@ -81,9 +82,9 @@ static void test_undo(void) { EXPECT(cst_undo_snapshot(g_ws, sid, "t3", "d/e/f.txt") == 0, "snap nested"); char p[512]; snprintf(p, sizeof(p), "%s/d", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); snprintf(p, sizeof(p), "%s/d/e", g_ws); - mkdir(p, 0755); + test_mkdir(p, 0755); wfile("d/e/f.txt", "x"); EXPECT(cst_undo_restore_last(g_ws, sid, rep, sizeof(rep)) == 0, "undo t3"); EXPECT(rfile_eq("d/e/f.txt", NULL), "nested file removed"); @@ -146,7 +147,7 @@ static void test_store(void) { EXPECT(cst_agent_store_path_for_id(g_ws, "20260101-000000-abcd", path, sizeof(path)) == 0, "path for id"); - FILE *fp = fopen(path, "w"); + FILE *fp = fopen(path, "wb"); EXPECT(fp != NULL, "create session file"); if (fp) { fputs("{\"role\":\"system\",\"content\":\"sys\"}\n", fp); @@ -174,17 +175,14 @@ static void test_store(void) { } int main(void) { - snprintf(g_ws, sizeof(g_ws), "/tmp/cst_test_undo_XXXXXX"); - if (!mkdtemp(g_ws)) { + if (test_tmpdir(g_ws, sizeof(g_ws), "cst_test_undo") != 0) { perror("mkdtemp"); return 1; } test_undo(); test_perms(); test_store(); - char cmd[512]; - snprintf(cmd, sizeof(cmd), "rm -rf '%s'", g_ws); - if (system(cmd) != 0) + if (test_rmrf(g_ws) != 0) fprintf(stderr, "warning: cleanup failed\n"); if (nfail) { fprintf(stderr, "test_undo: %d failure(s)\n", nfail); diff --git a/tests/test_win_utf8.c b/tests/test_win_utf8.c new file mode 100644 index 0000000..bd43e62 --- /dev/null +++ b/tests/test_win_utf8.c @@ -0,0 +1,238 @@ +/* UTF-8 everywhere: Chinese workspace and file names through the file + * tools and the shell, CRLF and BOM files edited by apply_patch, write_file + * and edit_file without losing their style, and (Windows) child output in + * the OEM code page turned into UTF-8. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "cst_agent.h" +#include "cst_util.h" +#include "os_compat.h" + +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "cst_test_util.h" + +static int nfail; + +#define EXPECT(cond, msg) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, msg); \ + nfail++; \ + } \ + } while (0) + +static char g_ws[1100]; + +/* tool(name, key, value, key, value, ..., NULL) */ +static int tool(cst_tool_result_t *r, const char *name, ...) { + ctl_tool_call_t c; + memset(&c, 0, sizeof(c)); + snprintf(c.id, sizeof(c.id), "t"); + snprintf(c.name, sizeof(c.name), "%s", name); + va_list ap; + va_start(ap, name); + const char *k; + while ((k = va_arg(ap, const char *)) != NULL) { + const char *v = va_arg(ap, const char *); + snprintf(c.args[c.nargs].key, sizeof(c.args[c.nargs].key), "%s", k); + c.args[c.nargs].value = (char *)v; + c.args[c.nargs].value_len = strlen(v); + c.nargs++; + } + va_end(ap); + memset(r, 0, sizeof(*r)); + cst_tool_dispatch(g_ws, &c, r); + return r->exit_code; +} + +static char *slurp(const char *rel, size_t *n) { + char p[1200]; + snprintf(p, sizeof(p), "%s/%s", g_ws, rel); + FILE *f = fopen(p, "rb"); + if (!f) + return NULL; + char *b = malloc(1 << 16); + *n = b ? fread(b, 1, (1 << 16) - 1, f) : 0; + if (b) + b[*n] = 0; + fclose(f); + return b; +} + +static void spit(const char *rel, const char *s) { + char p[1200]; + snprintf(p, sizeof(p), "%s/%s", g_ws, rel); + FILE *f = fopen(p, "wb"); + if (f) { + fputs(s, f); + fclose(f); + } +} + +static int file_is(const char *rel, const char *want) { + size_t n = 0; + char *b = slurp(rel, &n); + int ok = b && n == strlen(want) && memcmp(b, want, n) == 0; + if (!ok) + fprintf(stderr, " %s is \"%s\"\n", rel, b ? b : "(missing)"); + free(b); + return ok; +} + +static void test_patch_buffer(void) { + char *out = NULL, err[256]; + size_t n = 0; + int nh = 0; + const char *crlf = "a\r\nb\r\nc\r\n"; + const char *hunk = "@@ -1,3 +1,3 @@\n a\n-b\n+B\n c\n"; + EXPECT(cst_patch_apply_buffer(crlf, strlen(crlf), hunk, strlen(hunk), &out, + &n, &nh, err, sizeof(err)) == 0, + "LF hunk applies to a CRLF file"); + EXPECT(out && strcmp(out, "a\r\nB\r\nc\r\n") == 0, "CRLF kept"); + free(out); + + const char *bom = "\xEF\xBB\xBF" "first\nsecond\n"; + const char *h2 = "@@ -1,2 +1,2 @@\n-first\n+FIRST\n second\n"; + EXPECT(cst_patch_apply_buffer(bom, strlen(bom), h2, strlen(h2), &out, &n, + &nh, err, sizeof(err)) == 0, + "first line matches past the BOM"); + EXPECT(out && strcmp(out, "\xEF\xBB\xBF" "FIRST\nsecond\n") == 0, + "BOM kept"); + free(out); + + const char *lf = "x\ny\n"; + const char *h3 = "@@ -1,2 +1,3 @@\n x\n+中文\n y\n"; + EXPECT(cst_patch_apply_buffer(lf, strlen(lf), h3, strlen(h3), &out, &n, &nh, + err, sizeof(err)) == 0 && + out && strcmp(out, "x\n中文\ny\n") == 0, + "LF file stays LF"); + free(out); + + EXPECT(cst_text_is_crlf("a\r\nb\r\n", 6), "is_crlf"); + EXPECT(!cst_text_is_crlf("a\r\nb\n", 5), "mixed is not crlf"); + EXPECT(!cst_text_is_crlf("ab", 2), "no newline is not crlf"); +} + +static void test_tools(void) { + cst_tool_result_t r; + /* Chinese names through write/read/list/glob/grep */ + EXPECT(tool(&r, "write_file", "path", "目录/文件.txt", "content", + "你好,世界\n第二行\n", NULL) == 0, + "write_file with a Chinese path"); + cst_tool_result_free(&r); + EXPECT(tool(&r, "read_file", "path", "目录/文件.txt", NULL) == 0 && r.out && + strstr(r.out, "你好,世界"), + "read_file with a Chinese path"); + cst_tool_result_free(&r); + EXPECT(tool(&r, "list_dir", "path", "目录", NULL) == 0 && r.out && + strstr(r.out, "文件.txt"), + "list_dir shows the Chinese name"); + cst_tool_result_free(&r); + EXPECT(tool(&r, "glob", "pattern", "**/*.txt", NULL) == 0 && r.out && + strstr(r.out, "目录/文件.txt"), + "glob finds the Chinese name"); + cst_tool_result_free(&r); + EXPECT(tool(&r, "grep", "pattern", "世界", NULL) == 0 && r.out && + strstr(r.out, "文件.txt"), + "grep matches Chinese text"); + cst_tool_result_free(&r); + + /* CRLF + BOM file */ + spit("win.txt", "\xEF\xBB\xBFone\r\ntwo\r\nthree\r\n"); + EXPECT(tool(&r, "read_file", "path", "win.txt", NULL) == 0 && r.out && + strcmp(r.out, "one\ntwo\nthree\n") == 0, + "read_file hides BOM and CR"); + cst_tool_result_free(&r); + EXPECT(tool(&r, "edit_file", "path", "win.txt", "old_string", "two", + "new_string", "二\n2", NULL) == 0, + "edit_file single-line match"); + cst_tool_result_free(&r); + EXPECT(file_is("win.txt", "\xEF\xBB\xBFone\r\n二\r\n2\r\nthree\r\n"), + "edit_file inserts CRLF"); + EXPECT(tool(&r, "edit_file", "path", "win.txt", "old_string", "one\n二", + "new_string", "壹\n贰", NULL) == 0, + "edit_file multi-line LF match"); + cst_tool_result_free(&r); + EXPECT(file_is("win.txt", "\xEF\xBB\xBF壹\r\n贰\r\n2\r\nthree\r\n"), + "edit_file keeps CRLF and BOM"); + EXPECT(tool(&r, "apply_patch", "patch", + "--- a/win.txt\n+++ b/win.txt\n@@ -3,2 +3,2 @@\n 2\n-three\n" + "+叁\n", + NULL) == 0, + "apply_patch on a CRLF file"); + if (r.exit_code) + fprintf(stderr, " apply_patch: %s\n", r.out ? r.out : ""); + cst_tool_result_free(&r); + EXPECT(file_is("win.txt", "\xEF\xBB\xBF壹\r\n贰\r\n2\r\n叁\r\n"), + "apply_patch keeps CRLF and BOM"); + EXPECT(tool(&r, "write_file", "path", "win.txt", "content", "new\ntext\n", + NULL) == 0, + "write_file over a CRLF file"); + cst_tool_result_free(&r); + EXPECT(file_is("win.txt", "\xEF\xBB\xBFnew\r\ntext\r\n"), + "write_file keeps CRLF and BOM"); + EXPECT(tool(&r, "write_file", "path", "new.txt", "content", "lf\n", NULL) == + 0 && + file_is("new.txt", "lf\n"), + "new files are LF without BOM"); + cst_tool_result_free(&r); + + /* the shell sees and prints UTF-8 */ + EXPECT(tool(&r, "run_shell", "cmd", "cat 目录/文件.txt && echo 完成", NULL) == + 0 && + r.out && strstr(r.out, "你好,世界") && strstr(r.out, "完成"), + "run_shell with Chinese arguments and output"); + if (!r.out || !strstr(r.out, "完成")) + fprintf(stderr, " run_shell: %s\n", r.out ? r.out : ""); + cst_tool_result_free(&r); +} + +#ifdef _WIN32 +static void test_oem_output(void) { + /* "中文" in GBK */ + const char gbk[] = "\xD6\xD0\xCE\xC4\r\n"; + size_t n = 0; + char *u = os_output_to_utf8(gbk, sizeof(gbk) - 1, &n); + wchar_t b[8] = L"0"; + GetLocaleInfoW(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTCODEPAGE, b, 8); + unsigned oem = (unsigned)wcstoul(b, NULL, 10); + if (oem == 936) + EXPECT(u && strstr(u, "中文"), "GBK output decoded on a 936 system"); + else + EXPECT(u != NULL, "non-UTF-8 output is decoded"); + free(u); + EXPECT(os_output_to_utf8("中文", strlen("中文"), &n) == NULL, + "UTF-8 output left alone"); + printf("test_win_utf8: ACP %u, OEM %u, system OEM %u, console %u\n", + GetACP(), GetOEMCP(), oem, GetConsoleOutputCP()); + EXPECT(GetACP() == CP_UTF8, "manifest makes UTF-8 the process code page"); +} +#endif + +int main(void) { + os_console_init(); + char base[1024]; + if (test_tmpdir(base, sizeof(base), "cst-utf8") != 0) { + perror("tmpdir"); + return 1; + } + snprintf(g_ws, sizeof(g_ws), "%s/项目 空格", base); + test_mkdir(g_ws, 0755); + test_patch_buffer(); + test_tools(); +#ifdef _WIN32 + test_oem_output(); +#endif + test_rmrf(base); + if (nfail) { + fprintf(stderr, "test_win_utf8: %d failure(s)\n", nfail); + return 1; + } + puts("test_win_utf8: OK"); + return 0; +} diff --git a/third_party/busybox/LICENSE.busybox b/third_party/busybox/LICENSE.busybox index aa3aaca..ab2aa68 100644 --- a/third_party/busybox/LICENSE.busybox +++ b/third_party/busybox/LICENSE.busybox @@ -1,2 +1,3 @@ -BusyBox is GPLv2. Source: https://github.com/rmyorston/busybox-w32 -Upstream BusyBox: https://github.com/mirror/busybox +BusyBox is GPLv2. This busybox.exe is busybox-w32 (9b50a1e, Unicode build): +https://github.com/rmyorston/busybox-w32 +Upstream BusyBox: https://busybox.net/ diff --git a/vendor/tre-regex/COPYRIGHT.musl b/vendor/tre-regex/COPYRIGHT.musl new file mode 100644 index 0000000..c1628e9 --- /dev/null +++ b/vendor/tre-regex/COPYRIGHT.musl @@ -0,0 +1,193 @@ +musl as a whole is licensed under the following standard MIT license: + +---------------------------------------------------------------------- +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------------------------------------------------------------- + +Authors/contributors include: + +A. Wilcox +Ada Worcester +Alex Dowad +Alex Suykov +Alexander Monakov +Andre McCurdy +Andrew Kelley +Anthony G. Basile +Aric Belsito +Arvid Picciani +Bartosz Brachaczek +Benjamin Peterson +Bobby Bingham +Boris Brezillon +Brent Cook +Chris Spiegel +Clément Vasseur +Daniel Micay +Daniel Sabogal +Daurnimator +David Carlier +David Edelsohn +Denys Vlasenko +Dmitry Ivanov +Dmitry V. Levin +Drew DeVault +Emil Renner Berthing +Fangrui Song +Felix Fietkau +Felix Janda +Gianluca Anzolin +Hauke Mehrtens +He X +Hiltjo Posthuma +Isaac Dunham +Jaydeep Patil +Jens Gustedt +Jeremy Huntwork +Jo-Philipp Wich +Joakim Sindholt +John Spencer +Julien Ramseier +Justin Cormack +Kaarle Ritvanen +Khem Raj +Kylie McClain +Leah Neukirchen +Luca Barbato +Luka Perkov +M Farkas-Dyck (Strake) +Mahesh Bodapati +Markus Wichmann +Masanori Ogino +Michael Clark +Michael Forney +Mikhail Kremnyov +Natanael Copa +Nicholas J. Kain +orc +Pascal Cuoq +Patrick Oppenlander +Petr Hosek +Petr Skocik +Pierre Carrier +Reini Urban +Rich Felker +Richard Pennington +Ryan Fairfax +Samuel Holland +Segev Finer +Shiz +sin +Solar Designer +Stefan Kristiansson +Stefan O'Rear +Szabolcs Nagy +Timo Teräs +Trutz Behn +Valentin Ochs +Will Dietz +William Haddon +William Pitcock + +Portions of this software are derived from third-party works licensed +under terms compatible with the above MIT license: + +The TRE regular expression implementation (src/regex/reg* and +src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed +under a 2-clause BSD license (license text in the source files). The +included version has been heavily modified by Rich Felker in 2012, in +the interests of size, simplicity, and namespace cleanliness. + +Much of the math library code (src/math/* and src/complex/*) is +Copyright © 1993,2004 Sun Microsystems or +Copyright © 2003-2011 David Schultz or +Copyright © 2003-2009 Steven G. Kargl or +Copyright © 2003-2009 Bruce D. Evans or +Copyright © 2008 Stephen L. Moshier or +Copyright © 2017-2018 Arm Limited +and labelled as such in comments in the individual source files. All +have been licensed under extremely permissive terms. + +The ARM memcpy code (src/string/arm/memcpy.S) is Copyright © 2008 +The Android Open Source Project and is licensed under a two-clause BSD +license. It was taken from Bionic libc, used on Android. + +The AArch64 memcpy and memset code (src/string/aarch64/*) are +Copyright © 1999-2019, Arm Limited. + +The implementation of DES for crypt (src/crypt/crypt_des.c) is +Copyright © 1994 David Burren. It is licensed under a BSD license. + +The implementation of blowfish crypt (src/crypt/crypt_blowfish.c) was +originally written by Solar Designer and placed into the public +domain. The code also comes with a fallback permissive license for use +in jurisdictions that may not recognize the public domain. + +The smoothsort implementation (src/stdlib/qsort.c) is Copyright © 2011 +Valentin Ochs and is licensed under an MIT-style license. + +The x86_64 port was written by Nicholas J. Kain and is licensed under +the standard MIT terms. + +The mips and microblaze ports were originally written by Richard +Pennington for use in the ellcc project. The original code was adapted +by Rich Felker for build system and code conventions during upstream +integration. It is licensed under the standard MIT terms. + +The mips64 port was contributed by Imagination Technologies and is +licensed under the standard MIT terms. + +The powerpc port was also originally written by Richard Pennington, +and later supplemented and integrated by John Spencer. It is licensed +under the standard MIT terms. + +All other files which have no copyright comments are original works +produced specifically for use as part of this library, written either +by Rich Felker, the main author of the library, or by one or more +contibutors listed above. Details on authorship of individual files +can be found in the git version control history of the project. The +omission of copyright and license comments in each file is in the +interest of source tree size. + +In addition, permission is hereby granted for all public header files +(include/* and arch/*/bits/*) and crt files intended to be linked into +applications (crt/*, ldso/dlstart.c, and arch/*/crt_arch.h) to omit +the copyright notice and permission notice otherwise required by the +license, and to use these files without any requirement of +attribution. These files include substantial contributions from: + +Bobby Bingham +John Spencer +Nicholas J. Kain +Rich Felker +Richard Pennington +Stefan Kristiansson +Szabolcs Nagy + +all of whom have explicitly granted such permission. + +This file previously contained text expressing a belief that most of +the files covered by the above exception were sufficiently trivial not +to be subject to copyright, resulting in confusion over whether it +negated the permissions granted in the license. In the spirit of +permissive licensing, and of not having licensing issues being an +obstacle to adoption, that text has been removed. diff --git a/vendor/tre-regex/regcomp.c b/vendor/tre-regex/regcomp.c new file mode 100644 index 0000000..529fc4d --- /dev/null +++ b/vendor/tre-regex/regcomp.c @@ -0,0 +1,2953 @@ +/* + regcomp.c - TRE POSIX compatible regex compilation functions. + + Copyright (c) 2001-2009 Ville Laurikari <vl@iki.fi> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include <string.h> +#include <stdlib.h> +#include "regex.h" +#include <limits.h> +#include <stdint.h> +#include <ctype.h> + +#include "tre.h" + +#include <assert.h> + +/*********************************************************************** + from tre-compile.h +***********************************************************************/ + +typedef struct { + int position; + int code_min; + int code_max; + int *tags; + int assertions; + tre_ctype_t class; + tre_ctype_t *neg_classes; + int backref; +} tre_pos_and_tags_t; + + +/*********************************************************************** + from tre-ast.c and tre-ast.h +***********************************************************************/ + +/* The different AST node types. */ +typedef enum { + LITERAL, + CATENATION, + ITERATION, + UNION +} tre_ast_type_t; + +/* Special subtypes of TRE_LITERAL. */ +#define EMPTY -1 /* Empty leaf (denotes empty string). */ +#define ASSERTION -2 /* Assertion leaf. */ +#define TAG -3 /* Tag leaf. */ +#define BACKREF -4 /* Back reference leaf. */ + +#define IS_SPECIAL(x) ((x)->code_min < 0) +#define IS_EMPTY(x) ((x)->code_min == EMPTY) +#define IS_ASSERTION(x) ((x)->code_min == ASSERTION) +#define IS_TAG(x) ((x)->code_min == TAG) +#define IS_BACKREF(x) ((x)->code_min == BACKREF) + + +/* A generic AST node. All AST nodes consist of this node on the top + level with `obj' pointing to the actual content. */ +typedef struct { + tre_ast_type_t type; /* Type of the node. */ + void *obj; /* Pointer to actual node. */ + int nullable; + int submatch_id; + int num_submatches; + int num_tags; + tre_pos_and_tags_t *firstpos; + tre_pos_and_tags_t *lastpos; +} tre_ast_node_t; + + +/* A "literal" node. These are created for assertions, back references, + tags, matching parameter settings, and all expressions that match one + character. */ +typedef struct { + long code_min; + long code_max; + int position; + tre_ctype_t class; + tre_ctype_t *neg_classes; +} tre_literal_t; + +/* A "catenation" node. These are created when two regexps are concatenated. + If there are more than one subexpressions in sequence, the `left' part + holds all but the last, and `right' part holds the last subexpression + (catenation is left associative). */ +typedef struct { + tre_ast_node_t *left; + tre_ast_node_t *right; +} tre_catenation_t; + +/* An "iteration" node. These are created for the "*", "+", "?", and "{m,n}" + operators. */ +typedef struct { + /* Subexpression to match. */ + tre_ast_node_t *arg; + /* Minimum number of consecutive matches. */ + int min; + /* Maximum number of consecutive matches. */ + int max; + /* If 0, match as many characters as possible, if 1 match as few as + possible. Note that this does not always mean the same thing as + matching as many/few repetitions as possible. */ + unsigned int minimal:1; +} tre_iteration_t; + +/* An "union" node. These are created for the "|" operator. */ +typedef struct { + tre_ast_node_t *left; + tre_ast_node_t *right; +} tre_union_t; + + +static tre_ast_node_t * +tre_ast_new_node(tre_mem_t mem, int type, void *obj) +{ + tre_ast_node_t *node = tre_mem_calloc(mem, sizeof *node); + if (!node || !obj) + return 0; + node->obj = obj; + node->type = type; + node->nullable = -1; + node->submatch_id = -1; + return node; +} + +static tre_ast_node_t * +tre_ast_new_literal(tre_mem_t mem, int code_min, int code_max, int position) +{ + tre_ast_node_t *node; + tre_literal_t *lit; + + lit = tre_mem_calloc(mem, sizeof *lit); + node = tre_ast_new_node(mem, LITERAL, lit); + if (!node) + return 0; + lit->code_min = code_min; + lit->code_max = code_max; + lit->position = position; + return node; +} + +static tre_ast_node_t * +tre_ast_new_iter(tre_mem_t mem, tre_ast_node_t *arg, int min, int max, int minimal) +{ + tre_ast_node_t *node; + tre_iteration_t *iter; + + iter = tre_mem_calloc(mem, sizeof *iter); + node = tre_ast_new_node(mem, ITERATION, iter); + if (!node) + return 0; + iter->arg = arg; + iter->min = min; + iter->max = max; + iter->minimal = minimal; + node->num_submatches = arg->num_submatches; + return node; +} + +static tre_ast_node_t * +tre_ast_new_union(tre_mem_t mem, tre_ast_node_t *left, tre_ast_node_t *right) +{ + tre_ast_node_t *node; + tre_union_t *un; + + if (!left) + return right; + un = tre_mem_calloc(mem, sizeof *un); + node = tre_ast_new_node(mem, UNION, un); + if (!node || !right) + return 0; + un->left = left; + un->right = right; + node->num_submatches = left->num_submatches + right->num_submatches; + return node; +} + +static tre_ast_node_t * +tre_ast_new_catenation(tre_mem_t mem, tre_ast_node_t *left, tre_ast_node_t *right) +{ + tre_ast_node_t *node; + tre_catenation_t *cat; + + if (!left) + return right; + cat = tre_mem_calloc(mem, sizeof *cat); + node = tre_ast_new_node(mem, CATENATION, cat); + if (!node) + return 0; + cat->left = left; + cat->right = right; + node->num_submatches = left->num_submatches + right->num_submatches; + return node; +} + + +/*********************************************************************** + from tre-stack.c and tre-stack.h +***********************************************************************/ + +typedef struct tre_stack_rec tre_stack_t; + +/* Creates a new stack object. `size' is initial size in bytes, `max_size' + is maximum size, and `increment' specifies how much more space will be + allocated with realloc() if all space gets used up. Returns the stack + object or NULL if out of memory. */ +static tre_stack_t * +tre_stack_new(int size, int max_size, int increment); + +/* Frees the stack object. */ +static void +tre_stack_destroy(tre_stack_t *s); + +/* Returns the current number of objects in the stack. */ +static int +tre_stack_num_objects(tre_stack_t *s); + +/* Each tre_stack_push_*(tre_stack_t *s, <type> value) function pushes + `value' on top of stack `s'. Returns REG_ESPACE if out of memory. + This tries to realloc() more space before failing if maximum size + has not yet been reached. Returns REG_OK if successful. */ +#define declare_pushf(typetag, type) \ + static reg_errcode_t tre_stack_push_ ## typetag(tre_stack_t *s, type value) + +declare_pushf(voidptr, void *); +declare_pushf(int, int); + +/* Each tre_stack_pop_*(tre_stack_t *s) function pops the topmost + element off of stack `s' and returns it. The stack must not be + empty. */ +#define declare_popf(typetag, type) \ + static type tre_stack_pop_ ## typetag(tre_stack_t *s) + +declare_popf(voidptr, void *); +declare_popf(int, int); + +/* Just to save some typing. */ +#define STACK_PUSH(s, typetag, value) \ + do \ + { \ + status = tre_stack_push_ ## typetag(s, value); \ + } \ + while (/*CONSTCOND*/0) + +#define STACK_PUSHX(s, typetag, value) \ + { \ + status = tre_stack_push_ ## typetag(s, value); \ + if (status != REG_OK) \ + break; \ + } + +#define STACK_PUSHR(s, typetag, value) \ + { \ + reg_errcode_t _status; \ + _status = tre_stack_push_ ## typetag(s, value); \ + if (_status != REG_OK) \ + return _status; \ + } + +union tre_stack_item { + void *voidptr_value; + int int_value; +}; + +struct tre_stack_rec { + int size; + int max_size; + int increment; + int ptr; + union tre_stack_item *stack; +}; + + +static tre_stack_t * +tre_stack_new(int size, int max_size, int increment) +{ + tre_stack_t *s; + + s = xmalloc(sizeof(*s)); + if (s != NULL) + { + s->stack = xmalloc(sizeof(*s->stack) * size); + if (s->stack == NULL) + { + xfree(s); + return NULL; + } + s->size = size; + s->max_size = max_size; + s->increment = increment; + s->ptr = 0; + } + return s; +} + +static void +tre_stack_destroy(tre_stack_t *s) +{ + xfree(s->stack); + xfree(s); +} + +static int +tre_stack_num_objects(tre_stack_t *s) +{ + return s->ptr; +} + +static reg_errcode_t +tre_stack_push(tre_stack_t *s, union tre_stack_item value) +{ + if (s->ptr < s->size) + { + s->stack[s->ptr] = value; + s->ptr++; + } + else + { + if (s->size >= s->max_size) + { + return REG_ESPACE; + } + else + { + union tre_stack_item *new_buffer; + int new_size; + new_size = s->size + s->increment; + if (new_size > s->max_size) + new_size = s->max_size; + new_buffer = xrealloc(s->stack, sizeof(*new_buffer) * new_size); + if (new_buffer == NULL) + { + return REG_ESPACE; + } + assert(new_size > s->size); + s->size = new_size; + s->stack = new_buffer; + tre_stack_push(s, value); + } + } + return REG_OK; +} + +#define define_pushf(typetag, type) \ + declare_pushf(typetag, type) { \ + union tre_stack_item item; \ + item.typetag ## _value = value; \ + return tre_stack_push(s, item); \ +} + +define_pushf(int, int) +define_pushf(voidptr, void *) + +#define define_popf(typetag, type) \ + declare_popf(typetag, type) { \ + return s->stack[--s->ptr].typetag ## _value; \ + } + +define_popf(int, int) +define_popf(voidptr, void *) + + +/*********************************************************************** + from tre-parse.c and tre-parse.h +***********************************************************************/ + +/* Parse context. */ +typedef struct { + /* Memory allocator. The AST is allocated using this. */ + tre_mem_t mem; + /* Stack used for keeping track of regexp syntax. */ + tre_stack_t *stack; + /* The parsed node after a parse function returns. */ + tre_ast_node_t *n; + /* Position in the regexp pattern after a parse function returns. */ + const char *s; + /* The first character of the last subexpression parsed. */ + const char *start; + /* Current submatch ID. */ + int submatch_id; + /* Current position (number of literal). */ + int position; + /* The highest back reference or -1 if none seen so far. */ + int max_backref; + /* Compilation flags. */ + int cflags; +} tre_parse_ctx_t; + +/* Some macros for expanding \w, \s, etc. */ +static const struct { + char c; + const char *expansion; +} tre_macros[] = { + {'t', "\t"}, {'n', "\n"}, {'r', "\r"}, + {'f', "\f"}, {'a', "\a"}, {'e', "\033"}, + {'w', "[[:alnum:]_]"}, {'W', "[^[:alnum:]_]"}, {'s', "[[:space:]]"}, + {'S', "[^[:space:]]"}, {'d', "[[:digit:]]"}, {'D', "[^[:digit:]]"}, + { 0, 0 } +}; + +/* Expands a macro delimited by `regex' and `regex_end' to `buf', which + must have at least `len' items. Sets buf[0] to zero if the there + is no match in `tre_macros'. */ +static const char *tre_expand_macro(const char *s) +{ + int i; + for (i = 0; tre_macros[i].c && tre_macros[i].c != *s; i++); + return tre_macros[i].expansion; +} + +static int +tre_compare_lit(const void *a, const void *b) +{ + const tre_literal_t *const *la = a; + const tre_literal_t *const *lb = b; + /* assumes the range of valid code_min is < INT_MAX */ + return la[0]->code_min - lb[0]->code_min; +} + +struct literals { + tre_mem_t mem; + tre_literal_t **a; + int len; + int cap; +}; + +static tre_literal_t *tre_new_lit(struct literals *p) +{ + tre_literal_t **a; + if (p->len >= p->cap) { + if (p->cap >= 1<<15) + return 0; + p->cap *= 2; + a = xrealloc(p->a, p->cap * sizeof *p->a); + if (!a) + return 0; + p->a = a; + } + a = p->a + p->len++; + *a = tre_mem_calloc(p->mem, sizeof **a); + return *a; +} + +static int add_icase_literals(struct literals *ls, int min, int max) +{ + tre_literal_t *lit; + int b, e, c; + for (c=min; c<=max; ) { + /* assumes islower(c) and isupper(c) are exclusive + and toupper(c)!=c if islower(c). + multiple opposite case characters are not supported */ + if (tre_islower(c)) { + b = e = tre_toupper(c); + for (c++, e++; c<=max; c++, e++) + if (tre_toupper(c) != e) break; + } else if (tre_isupper(c)) { + b = e = tre_tolower(c); + for (c++, e++; c<=max; c++, e++) + if (tre_tolower(c) != e) break; + } else { + c++; + continue; + } + lit = tre_new_lit(ls); + if (!lit) + return -1; + lit->code_min = b; + lit->code_max = e-1; + lit->position = -1; + } + return 0; +} + + +/* Maximum number of character classes in a negated bracket expression. */ +#define MAX_NEG_CLASSES 64 + +struct neg { + int negate; + int len; + tre_ctype_t a[MAX_NEG_CLASSES]; +}; + +// TODO: parse bracket into a set of non-overlapping [lo,hi] ranges + +/* +bracket grammar: +Bracket = '[' List ']' | '[^' List ']' +List = Term | List Term +Term = Char | Range | Chclass | Eqclass +Range = Char '-' Char | Char '-' '-' +Char = Coll | coll_single +Meta = ']' | '-' +Coll = '[.' coll_single '.]' | '[.' coll_multi '.]' | '[.' Meta '.]' +Eqclass = '[=' coll_single '=]' | '[=' coll_multi '=]' +Chclass = '[:' class ':]' + +coll_single is a single char collating element but it can be + '-' only at the beginning or end of a List and + ']' only at the beginning of a List and + '^' anywhere except after the openning '[' +*/ + +static reg_errcode_t parse_bracket_terms(tre_parse_ctx_t *ctx, const char *s, struct literals *ls, struct neg *neg) +{ + const char *start = s; + tre_ctype_t class; + int min, max; + tre_char_t wc; + int len; + + for (;;) { + class = 0; + len = tre_mbtowc(&wc, s, 4); + if (len <= 0) + return *s ? REG_BADPAT : REG_EBRACK; + if (*s == ']' && s != start) { + ctx->s = s+1; + return REG_OK; + } + if (*s == '-' && s != start && s[1] != ']' && + /* extension: [a-z--@] is accepted as [a-z]|[--@] */ + (s[1] != '-' || s[2] == ']')) + return REG_ERANGE; + if (*s == '[' && (s[1] == '.' || s[1] == '=')) + /* collating symbols and equivalence classes are not supported */ + return REG_ECOLLATE; + if (*s == '[' && s[1] == ':') { + char tmp[CHARCLASS_NAME_MAX+1]; + s += 2; + for (len=0; len < CHARCLASS_NAME_MAX && s[len]; len++) { + if (s[len] == ':') { + memcpy(tmp, s, len); + tmp[len] = 0; + class = tre_ctype(tmp); + break; + } + } + if (!class || s[len+1] != ']') + return REG_ECTYPE; + min = 0; + max = TRE_CHAR_MAX; + s += len+2; + } else { + min = max = wc; + s += len; + if (*s == '-' && s[1] != ']') { + s++; + len = tre_mbtowc(&wc, s, 4); + max = wc; + /* XXX - Should use collation order instead of + encoding values in character ranges. */ + if (len <= 0 || min > max) + return REG_ERANGE; + s += len; + } + } + + if (class && neg->negate) { + if (neg->len >= MAX_NEG_CLASSES) + return REG_ESPACE; + neg->a[neg->len++] = class; + } else { + tre_literal_t *lit = tre_new_lit(ls); + if (!lit) + return REG_ESPACE; + lit->code_min = min; + lit->code_max = max; + lit->class = class; + lit->position = -1; + + /* Add opposite-case codepoints if REG_ICASE is present. + It seems that POSIX requires that bracket negation + should happen before case-folding, but most practical + implementations do it the other way around. Changing + the order would need efficient representation of + case-fold ranges and bracket range sets even with + simple patterns so this is ok for now. */ + if (ctx->cflags & REG_ICASE && !class) + if (add_icase_literals(ls, min, max)) + return REG_ESPACE; + } + } +} + +static reg_errcode_t parse_bracket(tre_parse_ctx_t *ctx, const char *s) +{ + int i, max, min, negmax, negmin; + tre_ast_node_t *node = 0, *n; + tre_ctype_t *nc = 0; + tre_literal_t *lit; + struct literals ls; + struct neg neg; + reg_errcode_t err; + + ls.mem = ctx->mem; + ls.len = 0; + ls.cap = 32; + ls.a = xmalloc(ls.cap * sizeof *ls.a); + if (!ls.a) + return REG_ESPACE; + neg.len = 0; + neg.negate = *s == '^'; + if (neg.negate) + s++; + + err = parse_bracket_terms(ctx, s, &ls, &neg); + if (err != REG_OK) + goto parse_bracket_done; + + if (neg.negate) { + /* + * With REG_NEWLINE, POSIX requires that newlines are not matched by + * any form of a non-matching list. + */ + if (ctx->cflags & REG_NEWLINE) { + lit = tre_new_lit(&ls); + if (!lit) { + err = REG_ESPACE; + goto parse_bracket_done; + } + lit->code_min = '\n'; + lit->code_max = '\n'; + lit->position = -1; + } + /* Sort the array if we need to negate it. */ + qsort(ls.a, ls.len, sizeof *ls.a, tre_compare_lit); + /* extra lit for the last negated range */ + lit = tre_new_lit(&ls); + if (!lit) { + err = REG_ESPACE; + goto parse_bracket_done; + } + lit->code_min = TRE_CHAR_MAX+1; + lit->code_max = TRE_CHAR_MAX+1; + lit->position = -1; + /* negated classes */ + if (neg.len) { + nc = tre_mem_alloc(ctx->mem, (neg.len+1)*sizeof *neg.a); + if (!nc) { + err = REG_ESPACE; + goto parse_bracket_done; + } + memcpy(nc, neg.a, neg.len*sizeof *neg.a); + nc[neg.len] = 0; + } + } + + /* Build a union of the items in the array, negated if necessary. */ + negmax = negmin = 0; + for (i = 0; i < ls.len; i++) { + lit = ls.a[i]; + min = lit->code_min; + max = lit->code_max; + if (neg.negate) { + if (min <= negmin) { + /* Overlap. */ + negmin = MAX(max + 1, negmin); + continue; + } + negmax = min - 1; + lit->code_min = negmin; + lit->code_max = negmax; + negmin = max + 1; + } + lit->position = ctx->position; + lit->neg_classes = nc; + n = tre_ast_new_node(ctx->mem, LITERAL, lit); + node = tre_ast_new_union(ctx->mem, node, n); + if (!node) { + err = REG_ESPACE; + break; + } + } + +parse_bracket_done: + xfree(ls.a); + ctx->position++; + ctx->n = node; + return err; +} + +static const char *parse_dup_count(const char *s, int *n) +{ + *n = -1; + if (!isdigit(*s)) + return s; + *n = 0; + for (;;) { + *n = 10 * *n + (*s - '0'); + s++; + if (!isdigit(*s) || *n > RE_DUP_MAX) + break; + } + return s; +} + +static const char *parse_dup(const char *s, int ere, int *pmin, int *pmax) +{ + int min, max; + + s = parse_dup_count(s, &min); + if (*s == ',') + s = parse_dup_count(s+1, &max); + else + max = min; + + if ( + (max < min && max >= 0) || + max > RE_DUP_MAX || + min > RE_DUP_MAX || + min < 0 || + (!ere && *s++ != '\\') || + *s++ != '}' + ) + return 0; + *pmin = min; + *pmax = max; + return s; +} + +static int hexval(unsigned c) +{ + if (c-'0'<10) return c-'0'; + c |= 32; + if (c-'a'<6) return c-'a'+10; + return -1; +} + +static reg_errcode_t marksub(tre_parse_ctx_t *ctx, tre_ast_node_t *node, int subid) +{ + if (node->submatch_id >= 0) { + tre_ast_node_t *n = tre_ast_new_literal(ctx->mem, EMPTY, -1, -1); + if (!n) + return REG_ESPACE; + n = tre_ast_new_catenation(ctx->mem, n, node); + if (!n) + return REG_ESPACE; + n->num_submatches = node->num_submatches; + node = n; + } + node->submatch_id = subid; + node->num_submatches++; + ctx->n = node; + return REG_OK; +} + +/* +BRE grammar: +Regex = Branch | '^' | '$' | '^$' | '^' Branch | Branch '$' | '^' Branch '$' +Branch = Atom | Branch Atom +Atom = char | quoted_char | '.' | Bracket | Atom Dup | '\(' Branch '\)' | back_ref +Dup = '*' | '\{' Count '\}' | '\{' Count ',\}' | '\{' Count ',' Count '\}' + +(leading ^ and trailing $ in a sub expr may be an anchor or literal as well) + +ERE grammar: +Regex = Branch | Regex '|' Branch +Branch = Atom | Branch Atom +Atom = char | quoted_char | '.' | Bracket | Atom Dup | '(' Regex ')' | '^' | '$' +Dup = '*' | '+' | '?' | '{' Count '}' | '{' Count ',}' | '{' Count ',' Count '}' + +(a*+?, ^*, $+, \X, {, (|a) are unspecified) +*/ + +static reg_errcode_t parse_atom(tre_parse_ctx_t *ctx, const char *s) +{ + int len, ere = ctx->cflags & REG_EXTENDED; + const char *p; + tre_ast_node_t *node; + tre_char_t wc; + switch (*s) { + case '[': + return parse_bracket(ctx, s+1); + case '\\': + p = tre_expand_macro(s+1); + if (p) { + /* assume \X expansion is a single atom */ + reg_errcode_t err = parse_atom(ctx, p); + ctx->s = s+2; + return err; + } + /* extensions: \b, \B, \<, \>, \xHH \x{HHHH} */ + switch (*++s) { + case 0: + return REG_EESCAPE; + case 'b': + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_WB, -1); + break; + case 'B': + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_WB_NEG, -1); + break; + case '<': + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_BOW, -1); + break; + case '>': + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_EOW, -1); + break; + case 'x': + s++; + int i, v = 0, c; + len = 2; + if (*s == '{') { + len = 8; + s++; + } + for (i=0; i<len && v<0x110000; i++) { + c = hexval(s[i]); + if (c < 0) break; + v = 16*v + c; + } + s += i; + if (len == 8) { + if (*s != '}') + return REG_EBRACE; + s++; + } + node = tre_ast_new_literal(ctx->mem, v, v, ctx->position++); + s--; + break; + case '{': + case '+': + case '?': + /* extension: treat \+, \? as repetitions in BRE */ + /* reject repetitions after empty expression in BRE */ + if (!ere) + return REG_BADRPT; + case '|': + /* extension: treat \| as alternation in BRE */ + if (!ere) { + node = tre_ast_new_literal(ctx->mem, EMPTY, -1, -1); + s--; + goto end; + } + /* fallthrough */ + default: + if (!ere && (unsigned)*s-'1' < 9) { + /* back reference */ + int val = *s - '0'; + node = tre_ast_new_literal(ctx->mem, BACKREF, val, ctx->position++); + ctx->max_backref = MAX(val, ctx->max_backref); + } else { + /* extension: accept unknown escaped char + as a literal */ + goto parse_literal; + } + } + s++; + break; + case '.': + if (ctx->cflags & REG_NEWLINE) { + tre_ast_node_t *tmp1, *tmp2; + tmp1 = tre_ast_new_literal(ctx->mem, 0, '\n'-1, ctx->position++); + tmp2 = tre_ast_new_literal(ctx->mem, '\n'+1, TRE_CHAR_MAX, ctx->position++); + if (tmp1 && tmp2) + node = tre_ast_new_union(ctx->mem, tmp1, tmp2); + else + node = 0; + } else { + node = tre_ast_new_literal(ctx->mem, 0, TRE_CHAR_MAX, ctx->position++); + } + s++; + break; + case '^': + /* '^' has a special meaning everywhere in EREs, and at beginning of BRE. */ + if (!ere && s != ctx->start) + goto parse_literal; + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_BOL, -1); + s++; + break; + case '$': + /* '$' is special everywhere in EREs, and at the end of a BRE subexpression. */ + if (!ere && s[1] && (s[1]!='\\'|| (s[2]!=')' && s[2]!='|'))) + goto parse_literal; + node = tre_ast_new_literal(ctx->mem, ASSERTION, ASSERT_AT_EOL, -1); + s++; + break; + case '*': + case '{': + case '+': + case '?': + /* reject repetitions after empty expression in ERE */ + if (ere) + return REG_BADRPT; + case '|': + if (!ere) + goto parse_literal; + case 0: + node = tre_ast_new_literal(ctx->mem, EMPTY, -1, -1); + break; + default: +parse_literal: + len = tre_mbtowc(&wc, s, 4); + if (len < 0) + return REG_BADPAT; + if (ctx->cflags & REG_ICASE && (tre_isupper(wc) || tre_islower(wc))) { + tre_ast_node_t *tmp1, *tmp2; + /* multiple opposite case characters are not supported */ + tmp1 = tre_ast_new_literal(ctx->mem, tre_toupper(wc), tre_toupper(wc), ctx->position); + tmp2 = tre_ast_new_literal(ctx->mem, tre_tolower(wc), tre_tolower(wc), ctx->position); + if (tmp1 && tmp2) + node = tre_ast_new_union(ctx->mem, tmp1, tmp2); + else + node = 0; + } else { + node = tre_ast_new_literal(ctx->mem, wc, wc, ctx->position); + } + ctx->position++; + s += len; + break; + } +end: + if (!node) + return REG_ESPACE; + ctx->n = node; + ctx->s = s; + return REG_OK; +} + +#define PUSHPTR(err, s, v) do { \ + if ((err = tre_stack_push_voidptr(s, v)) != REG_OK) \ + return err; \ +} while(0) + +#define PUSHINT(err, s, v) do { \ + if ((err = tre_stack_push_int(s, v)) != REG_OK) \ + return err; \ +} while(0) + +static reg_errcode_t tre_parse(tre_parse_ctx_t *ctx) +{ + tre_ast_node_t *nbranch=0, *nunion=0; + int ere = ctx->cflags & REG_EXTENDED; + const char *s = ctx->start; + int subid = 0; + int depth = 0; + reg_errcode_t err; + tre_stack_t *stack = ctx->stack; + + PUSHINT(err, stack, subid++); + for (;;) { + if ((!ere && *s == '\\' && s[1] == '(') || + (ere && *s == '(')) { + PUSHPTR(err, stack, nunion); + PUSHPTR(err, stack, nbranch); + PUSHINT(err, stack, subid++); + s++; + if (!ere) + s++; + depth++; + nbranch = nunion = 0; + ctx->start = s; + continue; + } + if ((!ere && *s == '\\' && s[1] == ')') || + (ere && *s == ')' && depth)) { + ctx->n = tre_ast_new_literal(ctx->mem, EMPTY, -1, -1); + if (!ctx->n) + return REG_ESPACE; + } else { + err = parse_atom(ctx, s); + if (err != REG_OK) + return err; + s = ctx->s; + } + + parse_iter: + for (;;) { + int min, max; + + if (*s!='\\' && *s!='*') { + if (!ere) + break; + if (*s!='+' && *s!='?' && *s!='{') + break; + } + if (*s=='\\' && ere) + break; + /* extension: treat \+, \? as repetitions in BRE */ + if (*s=='\\' && s[1]!='+' && s[1]!='?' && s[1]!='{') + break; + if (*s=='\\') + s++; + + /* handle ^* at the start of a BRE. */ + if (!ere && s==ctx->start+1 && s[-1]=='^') + break; + + /* extension: multiple consecutive *+?{,} is unspecified, + but (a+)+ has to be supported so accepting a++ makes + sense, note however that the RE_DUP_MAX limit can be + circumvented: (a{255}){255} uses a lot of memory.. */ + if (*s=='{') { + s = parse_dup(s+1, ere, &min, &max); + if (!s) + return REG_BADBR; + } else { + min=0; + max=-1; + if (*s == '+') + min = 1; + if (*s == '?') + max = 1; + s++; + } + if (max == 0) + ctx->n = tre_ast_new_literal(ctx->mem, EMPTY, -1, -1); + else + ctx->n = tre_ast_new_iter(ctx->mem, ctx->n, min, max, 0); + if (!ctx->n) + return REG_ESPACE; + } + + nbranch = tre_ast_new_catenation(ctx->mem, nbranch, ctx->n); + if ((ere && *s == '|') || + (ere && *s == ')' && depth) || + (!ere && *s == '\\' && s[1] == ')') || + /* extension: treat \| as alternation in BRE */ + (!ere && *s == '\\' && s[1] == '|') || + !*s) { + /* extension: empty branch is unspecified (), (|a), (a|) + here they are not rejected but match on empty string */ + int c = *s; + nunion = tre_ast_new_union(ctx->mem, nunion, nbranch); + nbranch = 0; + + if (c == '\\' && s[1] == '|') { + s+=2; + ctx->start = s; + } else if (c == '|') { + s++; + ctx->start = s; + } else { + if (c == '\\') { + if (!depth) return REG_EPAREN; + s+=2; + } else if (c == ')') + s++; + depth--; + err = marksub(ctx, nunion, tre_stack_pop_int(stack)); + if (err != REG_OK) + return err; + if (!c && depth<0) { + ctx->submatch_id = subid; + return REG_OK; + } + if (!c || depth<0) + return REG_EPAREN; + nbranch = tre_stack_pop_voidptr(stack); + nunion = tre_stack_pop_voidptr(stack); + goto parse_iter; + } + } + } +} + + +/*********************************************************************** + from tre-compile.c +***********************************************************************/ + + +/* + TODO: + - Fix tre_ast_to_tnfa() to recurse using a stack instead of recursive + function calls. +*/ + +/* + Algorithms to setup tags so that submatch addressing can be done. +*/ + + +/* Inserts a catenation node to the root of the tree given in `node'. + As the left child a new tag with number `tag_id' to `node' is added, + and the right child is the old root. */ +static reg_errcode_t +tre_add_tag_left(tre_mem_t mem, tre_ast_node_t *node, int tag_id) +{ + tre_catenation_t *c; + + c = tre_mem_alloc(mem, sizeof(*c)); + if (c == NULL) + return REG_ESPACE; + c->left = tre_ast_new_literal(mem, TAG, tag_id, -1); + if (c->left == NULL) + return REG_ESPACE; + c->right = tre_mem_alloc(mem, sizeof(tre_ast_node_t)); + if (c->right == NULL) + return REG_ESPACE; + + c->right->obj = node->obj; + c->right->type = node->type; + c->right->nullable = -1; + c->right->submatch_id = -1; + c->right->firstpos = NULL; + c->right->lastpos = NULL; + c->right->num_tags = 0; + c->right->num_submatches = 0; + node->obj = c; + node->type = CATENATION; + return REG_OK; +} + +/* Inserts a catenation node to the root of the tree given in `node'. + As the right child a new tag with number `tag_id' to `node' is added, + and the left child is the old root. */ +static reg_errcode_t +tre_add_tag_right(tre_mem_t mem, tre_ast_node_t *node, int tag_id) +{ + tre_catenation_t *c; + + c = tre_mem_alloc(mem, sizeof(*c)); + if (c == NULL) + return REG_ESPACE; + c->right = tre_ast_new_literal(mem, TAG, tag_id, -1); + if (c->right == NULL) + return REG_ESPACE; + c->left = tre_mem_alloc(mem, sizeof(tre_ast_node_t)); + if (c->left == NULL) + return REG_ESPACE; + + c->left->obj = node->obj; + c->left->type = node->type; + c->left->nullable = -1; + c->left->submatch_id = -1; + c->left->firstpos = NULL; + c->left->lastpos = NULL; + c->left->num_tags = 0; + c->left->num_submatches = 0; + node->obj = c; + node->type = CATENATION; + return REG_OK; +} + +typedef enum { + ADDTAGS_RECURSE, + ADDTAGS_AFTER_ITERATION, + ADDTAGS_AFTER_UNION_LEFT, + ADDTAGS_AFTER_UNION_RIGHT, + ADDTAGS_AFTER_CAT_LEFT, + ADDTAGS_AFTER_CAT_RIGHT, + ADDTAGS_SET_SUBMATCH_END +} tre_addtags_symbol_t; + + +typedef struct { + int tag; + int next_tag; +} tre_tag_states_t; + + +/* Go through `regset' and set submatch data for submatches that are + using this tag. */ +static void +tre_purge_regset(int *regset, tre_tnfa_t *tnfa, int tag) +{ + int i; + + for (i = 0; regset[i] >= 0; i++) + { + int id = regset[i] / 2; + int start = !(regset[i] % 2); + if (start) + tnfa->submatch_data[id].so_tag = tag; + else + tnfa->submatch_data[id].eo_tag = tag; + } + regset[0] = -1; +} + + +/* Adds tags to appropriate locations in the parse tree in `tree', so that + subexpressions marked for submatch addressing can be traced. */ +static reg_errcode_t +tre_add_tags(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *tree, + tre_tnfa_t *tnfa) +{ + reg_errcode_t status = REG_OK; + tre_addtags_symbol_t symbol; + tre_ast_node_t *node = tree; /* Tree node we are currently looking at. */ + int bottom = tre_stack_num_objects(stack); + /* True for first pass (counting number of needed tags) */ + int first_pass = (mem == NULL || tnfa == NULL); + int *regset, *orig_regset; + int num_tags = 0; /* Total number of tags. */ + int num_minimals = 0; /* Number of special minimal tags. */ + int tag = 0; /* The tag that is to be added next. */ + int next_tag = 1; /* Next tag to use after this one. */ + int *parents; /* Stack of submatches the current submatch is + contained in. */ + int minimal_tag = -1; /* Tag that marks the beginning of a minimal match. */ + tre_tag_states_t *saved_states; + + tre_tag_direction_t direction = TRE_TAG_MINIMIZE; + if (!first_pass) + { + tnfa->end_tag = 0; + tnfa->minimal_tags[0] = -1; + } + + regset = xmalloc(sizeof(*regset) * ((tnfa->num_submatches + 1) * 2)); + if (regset == NULL) + return REG_ESPACE; + regset[0] = -1; + orig_regset = regset; + + parents = xmalloc(sizeof(*parents) * (tnfa->num_submatches + 1)); + if (parents == NULL) + { + xfree(regset); + return REG_ESPACE; + } + parents[0] = -1; + + saved_states = xmalloc(sizeof(*saved_states) * (tnfa->num_submatches + 1)); + if (saved_states == NULL) + { + xfree(regset); + xfree(parents); + return REG_ESPACE; + } + else + { + unsigned int i; + for (i = 0; i <= tnfa->num_submatches; i++) + saved_states[i].tag = -1; + } + + STACK_PUSH(stack, voidptr, node); + STACK_PUSH(stack, int, ADDTAGS_RECURSE); + + while (tre_stack_num_objects(stack) > bottom) + { + if (status != REG_OK) + break; + + symbol = (tre_addtags_symbol_t)tre_stack_pop_int(stack); + switch (symbol) + { + + case ADDTAGS_SET_SUBMATCH_END: + { + int id = tre_stack_pop_int(stack); + int i; + + /* Add end of this submatch to regset. */ + for (i = 0; regset[i] >= 0; i++); + regset[i] = id * 2 + 1; + regset[i + 1] = -1; + + /* Pop this submatch from the parents stack. */ + for (i = 0; parents[i] >= 0; i++); + parents[i - 1] = -1; + break; + } + + case ADDTAGS_RECURSE: + node = tre_stack_pop_voidptr(stack); + + if (node->submatch_id >= 0) + { + int id = node->submatch_id; + int i; + + + /* Add start of this submatch to regset. */ + for (i = 0; regset[i] >= 0; i++); + regset[i] = id * 2; + regset[i + 1] = -1; + + if (!first_pass) + { + for (i = 0; parents[i] >= 0; i++); + tnfa->submatch_data[id].parents = NULL; + if (i > 0) + { + int *p = xmalloc(sizeof(*p) * (i + 1)); + if (p == NULL) + { + status = REG_ESPACE; + break; + } + assert(tnfa->submatch_data[id].parents == NULL); + tnfa->submatch_data[id].parents = p; + for (i = 0; parents[i] >= 0; i++) + p[i] = parents[i]; + p[i] = -1; + } + } + + /* Add end of this submatch to regset after processing this + node. */ + STACK_PUSHX(stack, int, node->submatch_id); + STACK_PUSHX(stack, int, ADDTAGS_SET_SUBMATCH_END); + } + + switch (node->type) + { + case LITERAL: + { + tre_literal_t *lit = node->obj; + + if (!IS_SPECIAL(lit) || IS_BACKREF(lit)) + { + int i; + if (regset[0] >= 0) + { + /* Regset is not empty, so add a tag before the + literal or backref. */ + if (!first_pass) + { + status = tre_add_tag_left(mem, node, tag); + tnfa->tag_directions[tag] = direction; + if (minimal_tag >= 0) + { + for (i = 0; tnfa->minimal_tags[i] >= 0; i++); + tnfa->minimal_tags[i] = tag; + tnfa->minimal_tags[i + 1] = minimal_tag; + tnfa->minimal_tags[i + 2] = -1; + minimal_tag = -1; + num_minimals++; + } + tre_purge_regset(regset, tnfa, tag); + } + else + { + node->num_tags = 1; + } + + regset[0] = -1; + tag = next_tag; + num_tags++; + next_tag++; + } + } + else + { + assert(!IS_TAG(lit)); + } + break; + } + case CATENATION: + { + tre_catenation_t *cat = node->obj; + tre_ast_node_t *left = cat->left; + tre_ast_node_t *right = cat->right; + int reserved_tag = -1; + + + /* After processing right child. */ + STACK_PUSHX(stack, voidptr, node); + STACK_PUSHX(stack, int, ADDTAGS_AFTER_CAT_RIGHT); + + /* Process right child. */ + STACK_PUSHX(stack, voidptr, right); + STACK_PUSHX(stack, int, ADDTAGS_RECURSE); + + /* After processing left child. */ + STACK_PUSHX(stack, int, next_tag + left->num_tags); + if (left->num_tags > 0 && right->num_tags > 0) + { + /* Reserve the next tag to the right child. */ + reserved_tag = next_tag; + next_tag++; + } + STACK_PUSHX(stack, int, reserved_tag); + STACK_PUSHX(stack, int, ADDTAGS_AFTER_CAT_LEFT); + + /* Process left child. */ + STACK_PUSHX(stack, voidptr, left); + STACK_PUSHX(stack, int, ADDTAGS_RECURSE); + + } + break; + case ITERATION: + { + tre_iteration_t *iter = node->obj; + + if (first_pass) + { + STACK_PUSHX(stack, int, regset[0] >= 0 || iter->minimal); + } + else + { + STACK_PUSHX(stack, int, tag); + STACK_PUSHX(stack, int, iter->minimal); + } + STACK_PUSHX(stack, voidptr, node); + STACK_PUSHX(stack, int, ADDTAGS_AFTER_ITERATION); + + STACK_PUSHX(stack, voidptr, iter->arg); + STACK_PUSHX(stack, int, ADDTAGS_RECURSE); + + /* Regset is not empty, so add a tag here. */ + if (regset[0] >= 0 || iter->minimal) + { + if (!first_pass) + { + int i; + status = tre_add_tag_left(mem, node, tag); + if (iter->minimal) + tnfa->tag_directions[tag] = TRE_TAG_MAXIMIZE; + else + tnfa->tag_directions[tag] = direction; + if (minimal_tag >= 0) + { + for (i = 0; tnfa->minimal_tags[i] >= 0; i++); + tnfa->minimal_tags[i] = tag; + tnfa->minimal_tags[i + 1] = minimal_tag; + tnfa->minimal_tags[i + 2] = -1; + minimal_tag = -1; + num_minimals++; + } + tre_purge_regset(regset, tnfa, tag); + } + + regset[0] = -1; + tag = next_tag; + num_tags++; + next_tag++; + } + direction = TRE_TAG_MINIMIZE; + } + break; + case UNION: + { + tre_union_t *uni = node->obj; + tre_ast_node_t *left = uni->left; + tre_ast_node_t *right = uni->right; + int left_tag; + int right_tag; + + if (regset[0] >= 0) + { + left_tag = next_tag; + right_tag = next_tag + 1; + } + else + { + left_tag = tag; + right_tag = next_tag; + } + + /* After processing right child. */ + STACK_PUSHX(stack, int, right_tag); + STACK_PUSHX(stack, int, left_tag); + STACK_PUSHX(stack, voidptr, regset); + STACK_PUSHX(stack, int, regset[0] >= 0); + STACK_PUSHX(stack, voidptr, node); + STACK_PUSHX(stack, voidptr, right); + STACK_PUSHX(stack, voidptr, left); + STACK_PUSHX(stack, int, ADDTAGS_AFTER_UNION_RIGHT); + + /* Process right child. */ + STACK_PUSHX(stack, voidptr, right); + STACK_PUSHX(stack, int, ADDTAGS_RECURSE); + + /* After processing left child. */ + STACK_PUSHX(stack, int, ADDTAGS_AFTER_UNION_LEFT); + + /* Process left child. */ + STACK_PUSHX(stack, voidptr, left); + STACK_PUSHX(stack, int, ADDTAGS_RECURSE); + + /* Regset is not empty, so add a tag here. */ + if (regset[0] >= 0) + { + if (!first_pass) + { + int i; + status = tre_add_tag_left(mem, node, tag); + tnfa->tag_directions[tag] = direction; + if (minimal_tag >= 0) + { + for (i = 0; tnfa->minimal_tags[i] >= 0; i++); + tnfa->minimal_tags[i] = tag; + tnfa->minimal_tags[i + 1] = minimal_tag; + tnfa->minimal_tags[i + 2] = -1; + minimal_tag = -1; + num_minimals++; + } + tre_purge_regset(regset, tnfa, tag); + } + + regset[0] = -1; + tag = next_tag; + num_tags++; + next_tag++; + } + + if (node->num_submatches > 0) + { + /* The next two tags are reserved for markers. */ + next_tag++; + tag = next_tag; + next_tag++; + } + + break; + } + } + + if (node->submatch_id >= 0) + { + int i; + /* Push this submatch on the parents stack. */ + for (i = 0; parents[i] >= 0; i++); + parents[i] = node->submatch_id; + parents[i + 1] = -1; + } + + break; /* end case: ADDTAGS_RECURSE */ + + case ADDTAGS_AFTER_ITERATION: + { + int minimal = 0; + int enter_tag; + node = tre_stack_pop_voidptr(stack); + if (first_pass) + { + node->num_tags = ((tre_iteration_t *)node->obj)->arg->num_tags + + tre_stack_pop_int(stack); + minimal_tag = -1; + } + else + { + minimal = tre_stack_pop_int(stack); + enter_tag = tre_stack_pop_int(stack); + if (minimal) + minimal_tag = enter_tag; + } + + if (!first_pass) + { + if (minimal) + direction = TRE_TAG_MINIMIZE; + else + direction = TRE_TAG_MAXIMIZE; + } + break; + } + + case ADDTAGS_AFTER_CAT_LEFT: + { + int new_tag = tre_stack_pop_int(stack); + next_tag = tre_stack_pop_int(stack); + if (new_tag >= 0) + { + tag = new_tag; + } + break; + } + + case ADDTAGS_AFTER_CAT_RIGHT: + node = tre_stack_pop_voidptr(stack); + if (first_pass) + node->num_tags = ((tre_catenation_t *)node->obj)->left->num_tags + + ((tre_catenation_t *)node->obj)->right->num_tags; + break; + + case ADDTAGS_AFTER_UNION_LEFT: + /* Lift the bottom of the `regset' array so that when processing + the right operand the items currently in the array are + invisible. The original bottom was saved at ADDTAGS_UNION and + will be restored at ADDTAGS_AFTER_UNION_RIGHT below. */ + while (*regset >= 0) + regset++; + break; + + case ADDTAGS_AFTER_UNION_RIGHT: + { + int added_tags, tag_left, tag_right; + tre_ast_node_t *left = tre_stack_pop_voidptr(stack); + tre_ast_node_t *right = tre_stack_pop_voidptr(stack); + node = tre_stack_pop_voidptr(stack); + added_tags = tre_stack_pop_int(stack); + if (first_pass) + { + node->num_tags = ((tre_union_t *)node->obj)->left->num_tags + + ((tre_union_t *)node->obj)->right->num_tags + added_tags + + ((node->num_submatches > 0) ? 2 : 0); + } + regset = tre_stack_pop_voidptr(stack); + tag_left = tre_stack_pop_int(stack); + tag_right = tre_stack_pop_int(stack); + + /* Add tags after both children, the left child gets a smaller + tag than the right child. This guarantees that we prefer + the left child over the right child. */ + /* XXX - This is not always necessary (if the children have + tags which must be seen for every match of that child). */ + /* XXX - Check if this is the only place where tre_add_tag_right + is used. If so, use tre_add_tag_left (putting the tag before + the child as opposed after the child) and throw away + tre_add_tag_right. */ + if (node->num_submatches > 0) + { + if (!first_pass) + { + status = tre_add_tag_right(mem, left, tag_left); + tnfa->tag_directions[tag_left] = TRE_TAG_MAXIMIZE; + if (status == REG_OK) + status = tre_add_tag_right(mem, right, tag_right); + tnfa->tag_directions[tag_right] = TRE_TAG_MAXIMIZE; + } + num_tags += 2; + } + direction = TRE_TAG_MAXIMIZE; + break; + } + + default: + assert(0); + break; + + } /* end switch(symbol) */ + } /* end while(tre_stack_num_objects(stack) > bottom) */ + + if (!first_pass) + tre_purge_regset(regset, tnfa, tag); + + if (!first_pass && minimal_tag >= 0) + { + int i; + for (i = 0; tnfa->minimal_tags[i] >= 0; i++); + tnfa->minimal_tags[i] = tag; + tnfa->minimal_tags[i + 1] = minimal_tag; + tnfa->minimal_tags[i + 2] = -1; + minimal_tag = -1; + num_minimals++; + } + + assert(tree->num_tags == num_tags); + tnfa->end_tag = num_tags; + tnfa->num_tags = num_tags; + tnfa->num_minimals = num_minimals; + xfree(orig_regset); + xfree(parents); + xfree(saved_states); + return status; +} + + + +/* + AST to TNFA compilation routines. +*/ + +typedef enum { + COPY_RECURSE, + COPY_SET_RESULT_PTR +} tre_copyast_symbol_t; + +/* Flags for tre_copy_ast(). */ +#define COPY_REMOVE_TAGS 1 +#define COPY_MAXIMIZE_FIRST_TAG 2 + +static reg_errcode_t +tre_copy_ast(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *ast, + int flags, int *pos_add, tre_tag_direction_t *tag_directions, + tre_ast_node_t **copy, int *max_pos) +{ + reg_errcode_t status = REG_OK; + int bottom = tre_stack_num_objects(stack); + int num_copied = 0; + int first_tag = 1; + tre_ast_node_t **result = copy; + tre_copyast_symbol_t symbol; + + STACK_PUSH(stack, voidptr, ast); + STACK_PUSH(stack, int, COPY_RECURSE); + + while (status == REG_OK && tre_stack_num_objects(stack) > bottom) + { + tre_ast_node_t *node; + if (status != REG_OK) + break; + + symbol = (tre_copyast_symbol_t)tre_stack_pop_int(stack); + switch (symbol) + { + case COPY_SET_RESULT_PTR: + result = tre_stack_pop_voidptr(stack); + break; + case COPY_RECURSE: + node = tre_stack_pop_voidptr(stack); + switch (node->type) + { + case LITERAL: + { + tre_literal_t *lit = node->obj; + int pos = lit->position; + int min = lit->code_min; + int max = lit->code_max; + if (!IS_SPECIAL(lit) || IS_BACKREF(lit)) + { + /* XXX - e.g. [ab] has only one position but two + nodes, so we are creating holes in the state space + here. Not fatal, just wastes memory. */ + pos += *pos_add; + num_copied++; + } + else if (IS_TAG(lit) && (flags & COPY_REMOVE_TAGS)) + { + /* Change this tag to empty. */ + min = EMPTY; + max = pos = -1; + } + else if (IS_TAG(lit) && (flags & COPY_MAXIMIZE_FIRST_TAG) + && first_tag) + { + /* Maximize the first tag. */ + tag_directions[max] = TRE_TAG_MAXIMIZE; + first_tag = 0; + } + *result = tre_ast_new_literal(mem, min, max, pos); + if (*result == NULL) + status = REG_ESPACE; + else { + tre_literal_t *p = (*result)->obj; + p->class = lit->class; + p->neg_classes = lit->neg_classes; + } + + if (pos > *max_pos) + *max_pos = pos; + break; + } + case UNION: + { + tre_union_t *uni = node->obj; + tre_union_t *tmp; + *result = tre_ast_new_union(mem, uni->left, uni->right); + if (*result == NULL) + { + status = REG_ESPACE; + break; + } + tmp = (*result)->obj; + result = &tmp->left; + STACK_PUSHX(stack, voidptr, uni->right); + STACK_PUSHX(stack, int, COPY_RECURSE); + STACK_PUSHX(stack, voidptr, &tmp->right); + STACK_PUSHX(stack, int, COPY_SET_RESULT_PTR); + STACK_PUSHX(stack, voidptr, uni->left); + STACK_PUSHX(stack, int, COPY_RECURSE); + break; + } + case CATENATION: + { + tre_catenation_t *cat = node->obj; + tre_catenation_t *tmp; + *result = tre_ast_new_catenation(mem, cat->left, cat->right); + if (*result == NULL) + { + status = REG_ESPACE; + break; + } + tmp = (*result)->obj; + tmp->left = NULL; + tmp->right = NULL; + result = &tmp->left; + + STACK_PUSHX(stack, voidptr, cat->right); + STACK_PUSHX(stack, int, COPY_RECURSE); + STACK_PUSHX(stack, voidptr, &tmp->right); + STACK_PUSHX(stack, int, COPY_SET_RESULT_PTR); + STACK_PUSHX(stack, voidptr, cat->left); + STACK_PUSHX(stack, int, COPY_RECURSE); + break; + } + case ITERATION: + { + tre_iteration_t *iter = node->obj; + STACK_PUSHX(stack, voidptr, iter->arg); + STACK_PUSHX(stack, int, COPY_RECURSE); + *result = tre_ast_new_iter(mem, iter->arg, iter->min, + iter->max, iter->minimal); + if (*result == NULL) + { + status = REG_ESPACE; + break; + } + iter = (*result)->obj; + result = &iter->arg; + break; + } + default: + assert(0); + break; + } + break; + } + } + *pos_add += num_copied; + return status; +} + +typedef enum { + EXPAND_RECURSE, + EXPAND_AFTER_ITER +} tre_expand_ast_symbol_t; + +/* Expands each iteration node that has a finite nonzero minimum or maximum + iteration count to a catenated sequence of copies of the node. */ +static reg_errcode_t +tre_expand_ast(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *ast, + int *position, tre_tag_direction_t *tag_directions) +{ + reg_errcode_t status = REG_OK; + int bottom = tre_stack_num_objects(stack); + int pos_add = 0; + int pos_add_total = 0; + int max_pos = 0; + int iter_depth = 0; + + STACK_PUSHR(stack, voidptr, ast); + STACK_PUSHR(stack, int, EXPAND_RECURSE); + while (status == REG_OK && tre_stack_num_objects(stack) > bottom) + { + tre_ast_node_t *node; + tre_expand_ast_symbol_t symbol; + + if (status != REG_OK) + break; + + symbol = (tre_expand_ast_symbol_t)tre_stack_pop_int(stack); + node = tre_stack_pop_voidptr(stack); + switch (symbol) + { + case EXPAND_RECURSE: + switch (node->type) + { + case LITERAL: + { + tre_literal_t *lit= node->obj; + if (!IS_SPECIAL(lit) || IS_BACKREF(lit)) + { + lit->position += pos_add; + if (lit->position > max_pos) + max_pos = lit->position; + } + break; + } + case UNION: + { + tre_union_t *uni = node->obj; + STACK_PUSHX(stack, voidptr, uni->right); + STACK_PUSHX(stack, int, EXPAND_RECURSE); + STACK_PUSHX(stack, voidptr, uni->left); + STACK_PUSHX(stack, int, EXPAND_RECURSE); + break; + } + case CATENATION: + { + tre_catenation_t *cat = node->obj; + STACK_PUSHX(stack, voidptr, cat->right); + STACK_PUSHX(stack, int, EXPAND_RECURSE); + STACK_PUSHX(stack, voidptr, cat->left); + STACK_PUSHX(stack, int, EXPAND_RECURSE); + break; + } + case ITERATION: + { + tre_iteration_t *iter = node->obj; + STACK_PUSHX(stack, int, pos_add); + STACK_PUSHX(stack, voidptr, node); + STACK_PUSHX(stack, int, EXPAND_AFTER_ITER); + STACK_PUSHX(stack, voidptr, iter->arg); + STACK_PUSHX(stack, int, EXPAND_RECURSE); + /* If we are going to expand this node at EXPAND_AFTER_ITER + then don't increase the `pos' fields of the nodes now, it + will get done when expanding. */ + if (iter->min > 1 || iter->max > 1) + pos_add = 0; + iter_depth++; + break; + } + default: + assert(0); + break; + } + break; + case EXPAND_AFTER_ITER: + { + tre_iteration_t *iter = node->obj; + int pos_add_last; + pos_add = tre_stack_pop_int(stack); + pos_add_last = pos_add; + if (iter->min > 1 || iter->max > 1) + { + tre_ast_node_t *seq1 = NULL, *seq2 = NULL; + int j; + int pos_add_save = pos_add; + + /* Create a catenated sequence of copies of the node. */ + for (j = 0; j < iter->min; j++) + { + tre_ast_node_t *copy; + /* Remove tags from all but the last copy. */ + int flags = ((j + 1 < iter->min) + ? COPY_REMOVE_TAGS + : COPY_MAXIMIZE_FIRST_TAG); + pos_add_save = pos_add; + status = tre_copy_ast(mem, stack, iter->arg, flags, + &pos_add, tag_directions, &copy, + &max_pos); + if (status != REG_OK) + return status; + if (seq1 != NULL) + seq1 = tre_ast_new_catenation(mem, seq1, copy); + else + seq1 = copy; + if (seq1 == NULL) + return REG_ESPACE; + } + + if (iter->max == -1) + { + /* No upper limit. */ + pos_add_save = pos_add; + status = tre_copy_ast(mem, stack, iter->arg, 0, + &pos_add, NULL, &seq2, &max_pos); + if (status != REG_OK) + return status; + seq2 = tre_ast_new_iter(mem, seq2, 0, -1, 0); + if (seq2 == NULL) + return REG_ESPACE; + } + else + { + for (j = iter->min; j < iter->max; j++) + { + tre_ast_node_t *tmp, *copy; + pos_add_save = pos_add; + status = tre_copy_ast(mem, stack, iter->arg, 0, + &pos_add, NULL, &copy, &max_pos); + if (status != REG_OK) + return status; + if (seq2 != NULL) + seq2 = tre_ast_new_catenation(mem, copy, seq2); + else + seq2 = copy; + if (seq2 == NULL) + return REG_ESPACE; + tmp = tre_ast_new_literal(mem, EMPTY, -1, -1); + if (tmp == NULL) + return REG_ESPACE; + seq2 = tre_ast_new_union(mem, tmp, seq2); + if (seq2 == NULL) + return REG_ESPACE; + } + } + + pos_add = pos_add_save; + if (seq1 == NULL) + seq1 = seq2; + else if (seq2 != NULL) + seq1 = tre_ast_new_catenation(mem, seq1, seq2); + if (seq1 == NULL) + return REG_ESPACE; + node->obj = seq1->obj; + node->type = seq1->type; + } + + iter_depth--; + pos_add_total += pos_add - pos_add_last; + if (iter_depth == 0) + pos_add = pos_add_total; + + break; + } + default: + assert(0); + break; + } + } + + *position += pos_add_total; + + /* `max_pos' should never be larger than `*position' if the above + code works, but just an extra safeguard let's make sure + `*position' is set large enough so enough memory will be + allocated for the transition table. */ + if (max_pos > *position) + *position = max_pos; + + return status; +} + +static tre_pos_and_tags_t * +tre_set_empty(tre_mem_t mem) +{ + tre_pos_and_tags_t *new_set; + + new_set = tre_mem_calloc(mem, sizeof(*new_set)); + if (new_set == NULL) + return NULL; + + new_set[0].position = -1; + new_set[0].code_min = -1; + new_set[0].code_max = -1; + + return new_set; +} + +static tre_pos_and_tags_t * +tre_set_one(tre_mem_t mem, int position, int code_min, int code_max, + tre_ctype_t class, tre_ctype_t *neg_classes, int backref) +{ + tre_pos_and_tags_t *new_set; + + new_set = tre_mem_calloc(mem, sizeof(*new_set) * 2); + if (new_set == NULL) + return NULL; + + new_set[0].position = position; + new_set[0].code_min = code_min; + new_set[0].code_max = code_max; + new_set[0].class = class; + new_set[0].neg_classes = neg_classes; + new_set[0].backref = backref; + new_set[1].position = -1; + new_set[1].code_min = -1; + new_set[1].code_max = -1; + + return new_set; +} + +static tre_pos_and_tags_t * +tre_set_union(tre_mem_t mem, tre_pos_and_tags_t *set1, tre_pos_and_tags_t *set2, + int *tags, int assertions) +{ + int s1, s2, i, j; + tre_pos_and_tags_t *new_set; + int *new_tags; + int num_tags; + + for (num_tags = 0; tags != NULL && tags[num_tags] >= 0; num_tags++); + for (s1 = 0; set1[s1].position >= 0; s1++); + for (s2 = 0; set2[s2].position >= 0; s2++); + new_set = tre_mem_calloc(mem, sizeof(*new_set) * (s1 + s2 + 1)); + if (!new_set ) + return NULL; + + for (s1 = 0; set1[s1].position >= 0; s1++) + { + new_set[s1].position = set1[s1].position; + new_set[s1].code_min = set1[s1].code_min; + new_set[s1].code_max = set1[s1].code_max; + new_set[s1].assertions = set1[s1].assertions | assertions; + new_set[s1].class = set1[s1].class; + new_set[s1].neg_classes = set1[s1].neg_classes; + new_set[s1].backref = set1[s1].backref; + if (set1[s1].tags == NULL && tags == NULL) + new_set[s1].tags = NULL; + else + { + for (i = 0; set1[s1].tags != NULL && set1[s1].tags[i] >= 0; i++); + new_tags = tre_mem_alloc(mem, (sizeof(*new_tags) + * (i + num_tags + 1))); + if (new_tags == NULL) + return NULL; + for (j = 0; j < i; j++) + new_tags[j] = set1[s1].tags[j]; + for (i = 0; i < num_tags; i++) + new_tags[j + i] = tags[i]; + new_tags[j + i] = -1; + new_set[s1].tags = new_tags; + } + } + + for (s2 = 0; set2[s2].position >= 0; s2++) + { + new_set[s1 + s2].position = set2[s2].position; + new_set[s1 + s2].code_min = set2[s2].code_min; + new_set[s1 + s2].code_max = set2[s2].code_max; + /* XXX - why not | assertions here as well? */ + new_set[s1 + s2].assertions = set2[s2].assertions; + new_set[s1 + s2].class = set2[s2].class; + new_set[s1 + s2].neg_classes = set2[s2].neg_classes; + new_set[s1 + s2].backref = set2[s2].backref; + if (set2[s2].tags == NULL) + new_set[s1 + s2].tags = NULL; + else + { + for (i = 0; set2[s2].tags[i] >= 0; i++); + new_tags = tre_mem_alloc(mem, sizeof(*new_tags) * (i + 1)); + if (new_tags == NULL) + return NULL; + for (j = 0; j < i; j++) + new_tags[j] = set2[s2].tags[j]; + new_tags[j] = -1; + new_set[s1 + s2].tags = new_tags; + } + } + new_set[s1 + s2].position = -1; + return new_set; +} + +/* Finds the empty path through `node' which is the one that should be + taken according to POSIX.2 rules, and adds the tags on that path to + `tags'. `tags' may be NULL. If `num_tags_seen' is not NULL, it is + set to the number of tags seen on the path. */ +static reg_errcode_t +tre_match_empty(tre_stack_t *stack, tre_ast_node_t *node, int *tags, + int *assertions, int *num_tags_seen) +{ + tre_literal_t *lit; + tre_union_t *uni; + tre_catenation_t *cat; + tre_iteration_t *iter; + int i; + int bottom = tre_stack_num_objects(stack); + reg_errcode_t status = REG_OK; + if (num_tags_seen) + *num_tags_seen = 0; + + status = tre_stack_push_voidptr(stack, node); + + /* Walk through the tree recursively. */ + while (status == REG_OK && tre_stack_num_objects(stack) > bottom) + { + node = tre_stack_pop_voidptr(stack); + + switch (node->type) + { + case LITERAL: + lit = (tre_literal_t *)node->obj; + switch (lit->code_min) + { + case TAG: + if (lit->code_max >= 0) + { + if (tags != NULL) + { + /* Add the tag to `tags'. */ + for (i = 0; tags[i] >= 0; i++) + if (tags[i] == lit->code_max) + break; + if (tags[i] < 0) + { + tags[i] = lit->code_max; + tags[i + 1] = -1; + } + } + if (num_tags_seen) + (*num_tags_seen)++; + } + break; + case ASSERTION: + assert(lit->code_max >= 1 + || lit->code_max <= ASSERT_LAST); + if (assertions != NULL) + *assertions |= lit->code_max; + break; + case EMPTY: + break; + default: + assert(0); + break; + } + break; + + case UNION: + /* Subexpressions starting earlier take priority over ones + starting later, so we prefer the left subexpression over the + right subexpression. */ + uni = (tre_union_t *)node->obj; + if (uni->left->nullable) + STACK_PUSHX(stack, voidptr, uni->left) + else if (uni->right->nullable) + STACK_PUSHX(stack, voidptr, uni->right) + else + assert(0); + break; + + case CATENATION: + /* The path must go through both children. */ + cat = (tre_catenation_t *)node->obj; + assert(cat->left->nullable); + assert(cat->right->nullable); + STACK_PUSHX(stack, voidptr, cat->left); + STACK_PUSHX(stack, voidptr, cat->right); + break; + + case ITERATION: + /* A match with an empty string is preferred over no match at + all, so we go through the argument if possible. */ + iter = (tre_iteration_t *)node->obj; + if (iter->arg->nullable) + STACK_PUSHX(stack, voidptr, iter->arg); + break; + + default: + assert(0); + break; + } + } + + return status; +} + + +typedef enum { + NFL_RECURSE, + NFL_POST_UNION, + NFL_POST_CATENATION, + NFL_POST_ITERATION +} tre_nfl_stack_symbol_t; + + +/* Computes and fills in the fields `nullable', `firstpos', and `lastpos' for + the nodes of the AST `tree'. */ +static reg_errcode_t +tre_compute_nfl(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *tree) +{ + int bottom = tre_stack_num_objects(stack); + + STACK_PUSHR(stack, voidptr, tree); + STACK_PUSHR(stack, int, NFL_RECURSE); + + while (tre_stack_num_objects(stack) > bottom) + { + tre_nfl_stack_symbol_t symbol; + tre_ast_node_t *node; + + symbol = (tre_nfl_stack_symbol_t)tre_stack_pop_int(stack); + node = tre_stack_pop_voidptr(stack); + switch (symbol) + { + case NFL_RECURSE: + switch (node->type) + { + case LITERAL: + { + tre_literal_t *lit = (tre_literal_t *)node->obj; + if (IS_BACKREF(lit)) + { + /* Back references: nullable = false, firstpos = {i}, + lastpos = {i}. */ + node->nullable = 0; + node->firstpos = tre_set_one(mem, lit->position, 0, + TRE_CHAR_MAX, 0, NULL, -1); + if (!node->firstpos) + return REG_ESPACE; + node->lastpos = tre_set_one(mem, lit->position, 0, + TRE_CHAR_MAX, 0, NULL, + (int)lit->code_max); + if (!node->lastpos) + return REG_ESPACE; + } + else if (lit->code_min < 0) + { + /* Tags, empty strings, params, and zero width assertions: + nullable = true, firstpos = {}, and lastpos = {}. */ + node->nullable = 1; + node->firstpos = tre_set_empty(mem); + if (!node->firstpos) + return REG_ESPACE; + node->lastpos = tre_set_empty(mem); + if (!node->lastpos) + return REG_ESPACE; + } + else + { + /* Literal at position i: nullable = false, firstpos = {i}, + lastpos = {i}. */ + node->nullable = 0; + node->firstpos = + tre_set_one(mem, lit->position, (int)lit->code_min, + (int)lit->code_max, 0, NULL, -1); + if (!node->firstpos) + return REG_ESPACE; + node->lastpos = tre_set_one(mem, lit->position, + (int)lit->code_min, + (int)lit->code_max, + lit->class, lit->neg_classes, + -1); + if (!node->lastpos) + return REG_ESPACE; + } + break; + } + + case UNION: + /* Compute the attributes for the two subtrees, and after that + for this node. */ + STACK_PUSHR(stack, voidptr, node); + STACK_PUSHR(stack, int, NFL_POST_UNION); + STACK_PUSHR(stack, voidptr, ((tre_union_t *)node->obj)->right); + STACK_PUSHR(stack, int, NFL_RECURSE); + STACK_PUSHR(stack, voidptr, ((tre_union_t *)node->obj)->left); + STACK_PUSHR(stack, int, NFL_RECURSE); + break; + + case CATENATION: + /* Compute the attributes for the two subtrees, and after that + for this node. */ + STACK_PUSHR(stack, voidptr, node); + STACK_PUSHR(stack, int, NFL_POST_CATENATION); + STACK_PUSHR(stack, voidptr, ((tre_catenation_t *)node->obj)->right); + STACK_PUSHR(stack, int, NFL_RECURSE); + STACK_PUSHR(stack, voidptr, ((tre_catenation_t *)node->obj)->left); + STACK_PUSHR(stack, int, NFL_RECURSE); + break; + + case ITERATION: + /* Compute the attributes for the subtree, and after that for + this node. */ + STACK_PUSHR(stack, voidptr, node); + STACK_PUSHR(stack, int, NFL_POST_ITERATION); + STACK_PUSHR(stack, voidptr, ((tre_iteration_t *)node->obj)->arg); + STACK_PUSHR(stack, int, NFL_RECURSE); + break; + } + break; /* end case: NFL_RECURSE */ + + case NFL_POST_UNION: + { + tre_union_t *uni = (tre_union_t *)node->obj; + node->nullable = uni->left->nullable || uni->right->nullable; + node->firstpos = tre_set_union(mem, uni->left->firstpos, + uni->right->firstpos, NULL, 0); + if (!node->firstpos) + return REG_ESPACE; + node->lastpos = tre_set_union(mem, uni->left->lastpos, + uni->right->lastpos, NULL, 0); + if (!node->lastpos) + return REG_ESPACE; + break; + } + + case NFL_POST_ITERATION: + { + tre_iteration_t *iter = (tre_iteration_t *)node->obj; + + if (iter->min == 0 || iter->arg->nullable) + node->nullable = 1; + else + node->nullable = 0; + node->firstpos = iter->arg->firstpos; + node->lastpos = iter->arg->lastpos; + break; + } + + case NFL_POST_CATENATION: + { + int num_tags, *tags, assertions; + reg_errcode_t status; + tre_catenation_t *cat = node->obj; + node->nullable = cat->left->nullable && cat->right->nullable; + + /* Compute firstpos. */ + if (cat->left->nullable) + { + /* The left side matches the empty string. Make a first pass + with tre_match_empty() to get the number of tags and + parameters. */ + status = tre_match_empty(stack, cat->left, + NULL, NULL, &num_tags); + if (status != REG_OK) + return status; + /* Allocate arrays for the tags and parameters. */ + tags = xmalloc(sizeof(*tags) * (num_tags + 1)); + if (!tags) + return REG_ESPACE; + tags[0] = -1; + assertions = 0; + /* Second pass with tre_mach_empty() to get the list of + tags and parameters. */ + status = tre_match_empty(stack, cat->left, tags, + &assertions, NULL); + if (status != REG_OK) + { + xfree(tags); + return status; + } + node->firstpos = + tre_set_union(mem, cat->right->firstpos, cat->left->firstpos, + tags, assertions); + xfree(tags); + if (!node->firstpos) + return REG_ESPACE; + } + else + { + node->firstpos = cat->left->firstpos; + } + + /* Compute lastpos. */ + if (cat->right->nullable) + { + /* The right side matches the empty string. Make a first pass + with tre_match_empty() to get the number of tags and + parameters. */ + status = tre_match_empty(stack, cat->right, + NULL, NULL, &num_tags); + if (status != REG_OK) + return status; + /* Allocate arrays for the tags and parameters. */ + tags = xmalloc(sizeof(int) * (num_tags + 1)); + if (!tags) + return REG_ESPACE; + tags[0] = -1; + assertions = 0; + /* Second pass with tre_mach_empty() to get the list of + tags and parameters. */ + status = tre_match_empty(stack, cat->right, tags, + &assertions, NULL); + if (status != REG_OK) + { + xfree(tags); + return status; + } + node->lastpos = + tre_set_union(mem, cat->left->lastpos, cat->right->lastpos, + tags, assertions); + xfree(tags); + if (!node->lastpos) + return REG_ESPACE; + } + else + { + node->lastpos = cat->right->lastpos; + } + break; + } + + default: + assert(0); + break; + } + } + + return REG_OK; +} + + +/* Adds a transition from each position in `p1' to each position in `p2'. */ +static reg_errcode_t +tre_make_trans(tre_pos_and_tags_t *p1, tre_pos_and_tags_t *p2, + tre_tnfa_transition_t *transitions, + int *counts, int *offs) +{ + tre_pos_and_tags_t *orig_p2 = p2; + tre_tnfa_transition_t *trans; + int i, j, k, l, dup, prev_p2_pos; + + if (transitions != NULL) + while (p1->position >= 0) + { + p2 = orig_p2; + prev_p2_pos = -1; + while (p2->position >= 0) + { + /* Optimization: if this position was already handled, skip it. */ + if (p2->position == prev_p2_pos) + { + p2++; + continue; + } + prev_p2_pos = p2->position; + /* Set `trans' to point to the next unused transition from + position `p1->position'. */ + trans = transitions + offs[p1->position]; + while (trans->state != NULL) + { +#if 0 + /* If we find a previous transition from `p1->position' to + `p2->position', it is overwritten. This can happen only + if there are nested loops in the regexp, like in "((a)*)*". + In POSIX.2 repetition using the outer loop is always + preferred over using the inner loop. Therefore the + transition for the inner loop is useless and can be thrown + away. */ + /* XXX - The same position is used for all nodes in a bracket + expression, so this optimization cannot be used (it will + break bracket expressions) unless I figure out a way to + detect it here. */ + if (trans->state_id == p2->position) + { + break; + } +#endif + trans++; + } + + if (trans->state == NULL) + (trans + 1)->state = NULL; + /* Use the character ranges, assertions, etc. from `p1' for + the transition from `p1' to `p2'. */ + trans->code_min = p1->code_min; + trans->code_max = p1->code_max; + trans->state = transitions + offs[p2->position]; + trans->state_id = p2->position; + trans->assertions = p1->assertions | p2->assertions + | (p1->class ? ASSERT_CHAR_CLASS : 0) + | (p1->neg_classes != NULL ? ASSERT_CHAR_CLASS_NEG : 0); + if (p1->backref >= 0) + { + assert((trans->assertions & ASSERT_CHAR_CLASS) == 0); + assert(p2->backref < 0); + trans->u.backref = p1->backref; + trans->assertions |= ASSERT_BACKREF; + } + else + trans->u.class = p1->class; + if (p1->neg_classes != NULL) + { + for (i = 0; p1->neg_classes[i] != (tre_ctype_t)0; i++); + trans->neg_classes = + xmalloc(sizeof(*trans->neg_classes) * (i + 1)); + if (trans->neg_classes == NULL) + return REG_ESPACE; + for (i = 0; p1->neg_classes[i] != (tre_ctype_t)0; i++) + trans->neg_classes[i] = p1->neg_classes[i]; + trans->neg_classes[i] = (tre_ctype_t)0; + } + else + trans->neg_classes = NULL; + + /* Find out how many tags this transition has. */ + i = 0; + if (p1->tags != NULL) + while(p1->tags[i] >= 0) + i++; + j = 0; + if (p2->tags != NULL) + while(p2->tags[j] >= 0) + j++; + + /* If we are overwriting a transition, free the old tag array. */ + if (trans->tags != NULL) + xfree(trans->tags); + trans->tags = NULL; + + /* If there were any tags, allocate an array and fill it. */ + if (i + j > 0) + { + trans->tags = xmalloc(sizeof(*trans->tags) * (i + j + 1)); + if (!trans->tags) + return REG_ESPACE; + i = 0; + if (p1->tags != NULL) + while(p1->tags[i] >= 0) + { + trans->tags[i] = p1->tags[i]; + i++; + } + l = i; + j = 0; + if (p2->tags != NULL) + while (p2->tags[j] >= 0) + { + /* Don't add duplicates. */ + dup = 0; + for (k = 0; k < i; k++) + if (trans->tags[k] == p2->tags[j]) + { + dup = 1; + break; + } + if (!dup) + trans->tags[l++] = p2->tags[j]; + j++; + } + trans->tags[l] = -1; + } + + p2++; + } + p1++; + } + else + /* Compute a maximum limit for the number of transitions leaving + from each state. */ + while (p1->position >= 0) + { + p2 = orig_p2; + while (p2->position >= 0) + { + counts[p1->position]++; + p2++; + } + p1++; + } + return REG_OK; +} + +/* Converts the syntax tree to a TNFA. All the transitions in the TNFA are + labelled with one character range (there are no transitions on empty + strings). The TNFA takes O(n^2) space in the worst case, `n' is size of + the regexp. */ +static reg_errcode_t +tre_ast_to_tnfa(tre_ast_node_t *node, tre_tnfa_transition_t *transitions, + int *counts, int *offs) +{ + tre_union_t *uni; + tre_catenation_t *cat; + tre_iteration_t *iter; + reg_errcode_t errcode = REG_OK; + + /* XXX - recurse using a stack!. */ + switch (node->type) + { + case LITERAL: + break; + case UNION: + uni = (tre_union_t *)node->obj; + errcode = tre_ast_to_tnfa(uni->left, transitions, counts, offs); + if (errcode != REG_OK) + return errcode; + errcode = tre_ast_to_tnfa(uni->right, transitions, counts, offs); + break; + + case CATENATION: + cat = (tre_catenation_t *)node->obj; + /* Add a transition from each position in cat->left->lastpos + to each position in cat->right->firstpos. */ + errcode = tre_make_trans(cat->left->lastpos, cat->right->firstpos, + transitions, counts, offs); + if (errcode != REG_OK) + return errcode; + errcode = tre_ast_to_tnfa(cat->left, transitions, counts, offs); + if (errcode != REG_OK) + return errcode; + errcode = tre_ast_to_tnfa(cat->right, transitions, counts, offs); + break; + + case ITERATION: + iter = (tre_iteration_t *)node->obj; + assert(iter->max == -1 || iter->max == 1); + + if (iter->max == -1) + { + assert(iter->min == 0 || iter->min == 1); + /* Add a transition from each last position in the iterated + expression to each first position. */ + errcode = tre_make_trans(iter->arg->lastpos, iter->arg->firstpos, + transitions, counts, offs); + if (errcode != REG_OK) + return errcode; + } + errcode = tre_ast_to_tnfa(iter->arg, transitions, counts, offs); + break; + } + return errcode; +} + + +#define ERROR_EXIT(err) \ + do \ + { \ + errcode = err; \ + if (/*CONSTCOND*/1) \ + goto error_exit; \ + } \ + while (/*CONSTCOND*/0) + + +int +regcomp(regex_t *restrict preg, const char *restrict regex, int cflags) +{ + tre_stack_t *stack; + tre_ast_node_t *tree, *tmp_ast_l, *tmp_ast_r; + tre_pos_and_tags_t *p; + int *counts = NULL, *offs = NULL; + int i, add = 0; + tre_tnfa_transition_t *transitions, *initial; + tre_tnfa_t *tnfa = NULL; + tre_submatch_data_t *submatch_data; + tre_tag_direction_t *tag_directions = NULL; + reg_errcode_t errcode; + tre_mem_t mem; + + /* Parse context. */ + tre_parse_ctx_t parse_ctx; + + /* Allocate a stack used throughout the compilation process for various + purposes. */ + stack = tre_stack_new(512, 1024000, 128); + if (!stack) + return REG_ESPACE; + /* Allocate a fast memory allocator. */ + mem = tre_mem_new(); + if (!mem) + { + tre_stack_destroy(stack); + return REG_ESPACE; + } + + /* Parse the regexp. */ + memset(&parse_ctx, 0, sizeof(parse_ctx)); + parse_ctx.mem = mem; + parse_ctx.stack = stack; + parse_ctx.start = regex; + parse_ctx.cflags = cflags; + parse_ctx.max_backref = -1; + errcode = tre_parse(&parse_ctx); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + preg->re_nsub = parse_ctx.submatch_id - 1; + tree = parse_ctx.n; + +#ifdef TRE_DEBUG + tre_ast_print(tree); +#endif /* TRE_DEBUG */ + + /* Referring to nonexistent subexpressions is illegal. */ + if (parse_ctx.max_backref > (int)preg->re_nsub) + ERROR_EXIT(REG_ESUBREG); + + /* Allocate the TNFA struct. */ + tnfa = xcalloc(1, sizeof(tre_tnfa_t)); + if (tnfa == NULL) + ERROR_EXIT(REG_ESPACE); + tnfa->have_backrefs = parse_ctx.max_backref >= 0; + tnfa->have_approx = 0; + tnfa->num_submatches = parse_ctx.submatch_id; + + /* Set up tags for submatch addressing. If REG_NOSUB is set and the + regexp does not have back references, this can be skipped. */ + if (tnfa->have_backrefs || !(cflags & REG_NOSUB)) + { + + /* Figure out how many tags we will need. */ + errcode = tre_add_tags(NULL, stack, tree, tnfa); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + + if (tnfa->num_tags > 0) + { + tag_directions = xmalloc(sizeof(*tag_directions) + * (tnfa->num_tags + 1)); + if (tag_directions == NULL) + ERROR_EXIT(REG_ESPACE); + tnfa->tag_directions = tag_directions; + memset(tag_directions, -1, + sizeof(*tag_directions) * (tnfa->num_tags + 1)); + } + tnfa->minimal_tags = xcalloc((unsigned)tnfa->num_tags * 2 + 1, + sizeof(*tnfa->minimal_tags)); + if (tnfa->minimal_tags == NULL) + ERROR_EXIT(REG_ESPACE); + + submatch_data = xcalloc((unsigned)parse_ctx.submatch_id, + sizeof(*submatch_data)); + if (submatch_data == NULL) + ERROR_EXIT(REG_ESPACE); + tnfa->submatch_data = submatch_data; + + errcode = tre_add_tags(mem, stack, tree, tnfa); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + + } + + /* Expand iteration nodes. */ + errcode = tre_expand_ast(mem, stack, tree, &parse_ctx.position, + tag_directions); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + + /* Add a dummy node for the final state. + XXX - For certain patterns this dummy node can be optimized away, + for example "a*" or "ab*". Figure out a simple way to detect + this possibility. */ + tmp_ast_l = tree; + tmp_ast_r = tre_ast_new_literal(mem, 0, 0, parse_ctx.position++); + if (tmp_ast_r == NULL) + ERROR_EXIT(REG_ESPACE); + + tree = tre_ast_new_catenation(mem, tmp_ast_l, tmp_ast_r); + if (tree == NULL) + ERROR_EXIT(REG_ESPACE); + + errcode = tre_compute_nfl(mem, stack, tree); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + + counts = xmalloc(sizeof(int) * parse_ctx.position); + if (counts == NULL) + ERROR_EXIT(REG_ESPACE); + + offs = xmalloc(sizeof(int) * parse_ctx.position); + if (offs == NULL) + ERROR_EXIT(REG_ESPACE); + + for (i = 0; i < parse_ctx.position; i++) + counts[i] = 0; + tre_ast_to_tnfa(tree, NULL, counts, NULL); + + add = 0; + for (i = 0; i < parse_ctx.position; i++) + { + offs[i] = add; + add += counts[i] + 1; + counts[i] = 0; + } + transitions = xcalloc((unsigned)add + 1, sizeof(*transitions)); + if (transitions == NULL) + ERROR_EXIT(REG_ESPACE); + tnfa->transitions = transitions; + tnfa->num_transitions = add; + + errcode = tre_ast_to_tnfa(tree, transitions, counts, offs); + if (errcode != REG_OK) + ERROR_EXIT(errcode); + + tnfa->firstpos_chars = NULL; + + p = tree->firstpos; + i = 0; + while (p->position >= 0) + { + i++; + p++; + } + + initial = xcalloc((unsigned)i + 1, sizeof(tre_tnfa_transition_t)); + if (initial == NULL) + ERROR_EXIT(REG_ESPACE); + tnfa->initial = initial; + + i = 0; + for (p = tree->firstpos; p->position >= 0; p++) + { + initial[i].state = transitions + offs[p->position]; + initial[i].state_id = p->position; + initial[i].tags = NULL; + /* Copy the arrays p->tags, and p->params, they are allocated + from a tre_mem object. */ + if (p->tags) + { + int j; + for (j = 0; p->tags[j] >= 0; j++); + initial[i].tags = xmalloc(sizeof(*p->tags) * (j + 1)); + if (!initial[i].tags) + ERROR_EXIT(REG_ESPACE); + memcpy(initial[i].tags, p->tags, sizeof(*p->tags) * (j + 1)); + } + initial[i].assertions = p->assertions; + i++; + } + initial[i].state = NULL; + + tnfa->num_transitions = add; + tnfa->final = transitions + offs[tree->lastpos[0].position]; + tnfa->num_states = parse_ctx.position; + tnfa->cflags = cflags; + + tre_mem_destroy(mem); + tre_stack_destroy(stack); + xfree(counts); + xfree(offs); + + preg->TRE_REGEX_T_FIELD = (void *)tnfa; + return REG_OK; + + error_exit: + /* Free everything that was allocated and return the error code. */ + tre_mem_destroy(mem); + if (stack != NULL) + tre_stack_destroy(stack); + if (counts != NULL) + xfree(counts); + if (offs != NULL) + xfree(offs); + preg->TRE_REGEX_T_FIELD = (void *)tnfa; + regfree(preg); + return errcode; +} + + + + +void +regfree(regex_t *preg) +{ + tre_tnfa_t *tnfa; + unsigned int i; + tre_tnfa_transition_t *trans; + + tnfa = (void *)preg->TRE_REGEX_T_FIELD; + if (!tnfa) + return; + + for (i = 0; i < tnfa->num_transitions; i++) + if (tnfa->transitions[i].state) + { + if (tnfa->transitions[i].tags) + xfree(tnfa->transitions[i].tags); + if (tnfa->transitions[i].neg_classes) + xfree(tnfa->transitions[i].neg_classes); + } + if (tnfa->transitions) + xfree(tnfa->transitions); + + if (tnfa->initial) + { + for (trans = tnfa->initial; trans->state; trans++) + { + if (trans->tags) + xfree(trans->tags); + } + xfree(tnfa->initial); + } + + if (tnfa->submatch_data) + { + for (i = 0; i < tnfa->num_submatches; i++) + if (tnfa->submatch_data[i].parents) + xfree(tnfa->submatch_data[i].parents); + xfree(tnfa->submatch_data); + } + + if (tnfa->tag_directions) + xfree(tnfa->tag_directions); + if (tnfa->firstpos_chars) + xfree(tnfa->firstpos_chars); + if (tnfa->minimal_tags) + xfree(tnfa->minimal_tags); + xfree(tnfa); +} diff --git a/vendor/tre-regex/regerror.c b/vendor/tre-regex/regerror.c new file mode 100644 index 0000000..a3ebe21 --- /dev/null +++ b/vendor/tre-regex/regerror.c @@ -0,0 +1,36 @@ +#include <string.h> +#include "regex.h" +#include <stdio.h> + +/* Error message strings for error codes listed in `regex.h'. This list + needs to be in sync with the codes listed there, naturally. */ + +/* Converted to single string by Rich Felker to remove the need for + * data relocations at runtime, 27 Feb 2006. */ + +static const char messages[] = { + "No error\0" + "No match\0" + "Invalid regexp\0" + "Unknown collating element\0" + "Unknown character class name\0" + "Trailing backslash\0" + "Invalid back reference\0" + "Missing ']'\0" + "Missing ')'\0" + "Missing '}'\0" + "Invalid contents of {}\0" + "Invalid character range\0" + "Out of memory\0" + "Repetition not preceded by valid expression\0" + "\0Unknown error" +}; + +size_t regerror(int e, const regex_t *restrict preg, char *restrict buf, size_t size) +{ + const char *s; + (void)preg; + for (s=messages; e && *s; e--, s+=strlen(s)+1); + if (!*s) s++; + return 1+snprintf(buf, size, "%s", s); +} diff --git a/vendor/tre-regex/regex.h b/vendor/tre-regex/regex.h new file mode 100644 index 0000000..73bb431 --- /dev/null +++ b/vendor/tre-regex/regex.h @@ -0,0 +1,62 @@ +/* POSIX <regex.h> for platforms without one (MinGW). The engine is musl's + * TRE (vendor/tre-regex, musl 1.2.5) patched to decode UTF-8 itself, so + * patterns and subjects are UTF-8 regardless of wchar_t width. */ +#ifndef _REGEX_H +#define _REGEX_H + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef ptrdiff_t regoff_t; + +typedef struct re_pattern_buffer { + size_t re_nsub; + void *__opaque, *__padding[4]; + size_t __nsub2; + char __padding2; +} regex_t; + +typedef struct { + regoff_t rm_so; + regoff_t rm_eo; +} regmatch_t; + +#define REG_EXTENDED 1 +#define REG_ICASE 2 +#define REG_NEWLINE 4 +#define REG_NOSUB 8 + +#define REG_NOTBOL 1 +#define REG_NOTEOL 2 + +#define REG_OK 0 +#define REG_NOMATCH 1 +#define REG_BADPAT 2 +#define REG_ECOLLATE 3 +#define REG_ECTYPE 4 +#define REG_EESCAPE 5 +#define REG_ESUBREG 6 +#define REG_EBRACK 7 +#define REG_EPAREN 8 +#define REG_EBRACE 9 +#define REG_BADBR 10 +#define REG_ERANGE 11 +#define REG_ESPACE 12 +#define REG_BADRPT 13 + +#define REG_ENOSYS -1 + +int regcomp(regex_t *__restrict, const char *__restrict, int); +int regexec(const regex_t *__restrict, const char *__restrict, size_t, regmatch_t *__restrict, int); +void regfree(regex_t *); + +size_t regerror(int, const regex_t *__restrict, char *__restrict, size_t); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/vendor/tre-regex/regexec.c b/vendor/tre-regex/regexec.c new file mode 100644 index 0000000..e0fcd8a --- /dev/null +++ b/vendor/tre-regex/regexec.c @@ -0,0 +1,1028 @@ +/* + regexec.c - TRE POSIX compatible matching functions (and more). + + Copyright (c) 2001-2009 Ville Laurikari <vl@iki.fi> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include <stdlib.h> +#include <string.h> +#include <wchar.h> +#include <wctype.h> +#include <limits.h> +#include <stdint.h> + +#include "regex.h" + +#include "tre.h" + +#include <assert.h> + +static void +tre_fill_pmatch(size_t nmatch, regmatch_t pmatch[], int cflags, + const tre_tnfa_t *tnfa, regoff_t *tags, regoff_t match_eo); + +/*********************************************************************** + from tre-match-utils.h +***********************************************************************/ + +#define GET_NEXT_WCHAR() do { \ + prev_c = next_c; pos += pos_add_next; \ + if ((pos_add_next = tre_mbtowc(&next_c, str_byte, 4)) <= 0) { \ + if (pos_add_next < 0) { ret = REG_NOMATCH; goto error_exit; } \ + else pos_add_next++; \ + } \ + str_byte += pos_add_next; \ + } while (0) + +#define IS_WORD_CHAR(c) ((c) == L'_' || tre_isalnum(c)) + +#define CHECK_ASSERTIONS(assertions) \ + (((assertions & ASSERT_AT_BOL) \ + && (pos > 0 || reg_notbol) \ + && (prev_c != L'\n' || !reg_newline)) \ + || ((assertions & ASSERT_AT_EOL) \ + && (next_c != L'\0' || reg_noteol) \ + && (next_c != L'\n' || !reg_newline)) \ + || ((assertions & ASSERT_AT_BOW) \ + && (IS_WORD_CHAR(prev_c) || !IS_WORD_CHAR(next_c))) \ + || ((assertions & ASSERT_AT_EOW) \ + && (!IS_WORD_CHAR(prev_c) || IS_WORD_CHAR(next_c))) \ + || ((assertions & ASSERT_AT_WB) \ + && (pos != 0 && next_c != L'\0' \ + && IS_WORD_CHAR(prev_c) == IS_WORD_CHAR(next_c))) \ + || ((assertions & ASSERT_AT_WB_NEG) \ + && (pos == 0 || next_c == L'\0' \ + || IS_WORD_CHAR(prev_c) != IS_WORD_CHAR(next_c)))) + +#define CHECK_CHAR_CLASSES(trans_i, tnfa, eflags) \ + (((trans_i->assertions & ASSERT_CHAR_CLASS) \ + && !(tnfa->cflags & REG_ICASE) \ + && !tre_isctype((tre_cint_t)prev_c, trans_i->u.class)) \ + || ((trans_i->assertions & ASSERT_CHAR_CLASS) \ + && (tnfa->cflags & REG_ICASE) \ + && !tre_isctype(tre_tolower((tre_cint_t)prev_c),trans_i->u.class) \ + && !tre_isctype(tre_toupper((tre_cint_t)prev_c),trans_i->u.class)) \ + || ((trans_i->assertions & ASSERT_CHAR_CLASS_NEG) \ + && tre_neg_char_classes_match(trans_i->neg_classes,(tre_cint_t)prev_c,\ + tnfa->cflags & REG_ICASE))) + + + + +/* Returns 1 if `t1' wins `t2', 0 otherwise. */ +static int +tre_tag_order(int num_tags, tre_tag_direction_t *tag_directions, + regoff_t *t1, regoff_t *t2) +{ + int i; + for (i = 0; i < num_tags; i++) + { + if (tag_directions[i] == TRE_TAG_MINIMIZE) + { + if (t1[i] < t2[i]) + return 1; + if (t1[i] > t2[i]) + return 0; + } + else + { + if (t1[i] > t2[i]) + return 1; + if (t1[i] < t2[i]) + return 0; + } + } + /* assert(0);*/ + return 0; +} + +static int +tre_neg_char_classes_match(tre_ctype_t *classes, tre_cint_t wc, int icase) +{ + while (*classes != (tre_ctype_t)0) + if ((!icase && tre_isctype(wc, *classes)) + || (icase && (tre_isctype(tre_toupper(wc), *classes) + || tre_isctype(tre_tolower(wc), *classes)))) + return 1; /* Match. */ + else + classes++; + return 0; /* No match. */ +} + + +/*********************************************************************** + from tre-match-parallel.c +***********************************************************************/ + +/* + This algorithm searches for matches basically by reading characters + in the searched string one by one, starting at the beginning. All + matching paths in the TNFA are traversed in parallel. When two or + more paths reach the same state, exactly one is chosen according to + tag ordering rules; if returning submatches is not required it does + not matter which path is chosen. + + The worst case time required for finding the leftmost and longest + match, or determining that there is no match, is always linearly + dependent on the length of the text being searched. + + This algorithm cannot handle TNFAs with back referencing nodes. + See `tre-match-backtrack.c'. +*/ + +typedef struct { + tre_tnfa_transition_t *state; + regoff_t *tags; +} tre_tnfa_reach_t; + +typedef struct { + regoff_t pos; + regoff_t **tags; +} tre_reach_pos_t; + + +static reg_errcode_t +tre_tnfa_run_parallel(const tre_tnfa_t *tnfa, const void *string, + regoff_t *match_tags, int eflags, + regoff_t *match_end_ofs) +{ + /* State variables required by GET_NEXT_WCHAR. */ + tre_char_t prev_c = 0, next_c = 0; + const char *str_byte = string; + regoff_t pos = -1; + regoff_t pos_add_next = 1; +#ifdef TRE_MBSTATE + mbstate_t mbstate; +#endif /* TRE_MBSTATE */ + int reg_notbol = eflags & REG_NOTBOL; + int reg_noteol = eflags & REG_NOTEOL; + int reg_newline = tnfa->cflags & REG_NEWLINE; + reg_errcode_t ret; + + char *buf; + tre_tnfa_transition_t *trans_i; + tre_tnfa_reach_t *reach, *reach_next, *reach_i, *reach_next_i; + tre_reach_pos_t *reach_pos; + int *tag_i; + int num_tags, i; + + regoff_t match_eo = -1; /* end offset of match (-1 if no match found yet) */ + int new_match = 0; + regoff_t *tmp_tags = NULL; + regoff_t *tmp_iptr; + +#ifdef TRE_MBSTATE + memset(&mbstate, '\0', sizeof(mbstate)); +#endif /* TRE_MBSTATE */ + + if (!match_tags) + num_tags = 0; + else + num_tags = tnfa->num_tags; + + /* Allocate memory for temporary data required for matching. This needs to + be done for every matching operation to be thread safe. This allocates + everything in a single large block with calloc(). */ + { + size_t tbytes, rbytes, pbytes, xbytes, total_bytes; + char *tmp_buf; + + /* Ensure that tbytes and xbytes*num_states cannot overflow, and that + * they don't contribute more than 1/8 of SIZE_MAX to total_bytes. */ + if (num_tags > SIZE_MAX/(8 * sizeof(regoff_t) * tnfa->num_states)) + return REG_ESPACE; + + /* Likewise check rbytes. */ + if (tnfa->num_states+1 > SIZE_MAX/(8 * sizeof(*reach_next))) + return REG_ESPACE; + + /* Likewise check pbytes. */ + if (tnfa->num_states > SIZE_MAX/(8 * sizeof(*reach_pos))) + return REG_ESPACE; + + /* Compute the length of the block we need. */ + tbytes = sizeof(*tmp_tags) * num_tags; + rbytes = sizeof(*reach_next) * (tnfa->num_states + 1); + pbytes = sizeof(*reach_pos) * tnfa->num_states; + xbytes = sizeof(regoff_t) * num_tags; + total_bytes = + (sizeof(long) - 1) * 4 /* for alignment paddings */ + + (rbytes + xbytes * tnfa->num_states) * 2 + tbytes + pbytes; + + /* Allocate the memory. */ + buf = calloc(total_bytes, 1); + if (buf == NULL) + return REG_ESPACE; + + /* Get the various pointers within tmp_buf (properly aligned). */ + tmp_tags = (void *)buf; + tmp_buf = buf + tbytes; + tmp_buf += ALIGN(tmp_buf, long); + reach_next = (void *)tmp_buf; + tmp_buf += rbytes; + tmp_buf += ALIGN(tmp_buf, long); + reach = (void *)tmp_buf; + tmp_buf += rbytes; + tmp_buf += ALIGN(tmp_buf, long); + reach_pos = (void *)tmp_buf; + tmp_buf += pbytes; + tmp_buf += ALIGN(tmp_buf, long); + for (i = 0; i < tnfa->num_states; i++) + { + reach[i].tags = (void *)tmp_buf; + tmp_buf += xbytes; + reach_next[i].tags = (void *)tmp_buf; + tmp_buf += xbytes; + } + } + + for (i = 0; i < tnfa->num_states; i++) + reach_pos[i].pos = -1; + + GET_NEXT_WCHAR(); + pos = 0; + + reach_next_i = reach_next; + while (1) + { + /* If no match found yet, add the initial states to `reach_next'. */ + if (match_eo < 0) + { + trans_i = tnfa->initial; + while (trans_i->state != NULL) + { + if (reach_pos[trans_i->state_id].pos < pos) + { + if (trans_i->assertions + && CHECK_ASSERTIONS(trans_i->assertions)) + { + trans_i++; + continue; + } + + reach_next_i->state = trans_i->state; + for (i = 0; i < num_tags; i++) + reach_next_i->tags[i] = -1; + tag_i = trans_i->tags; + if (tag_i) + while (*tag_i >= 0) + { + if (*tag_i < num_tags) + reach_next_i->tags[*tag_i] = pos; + tag_i++; + } + if (reach_next_i->state == tnfa->final) + { + match_eo = pos; + new_match = 1; + for (i = 0; i < num_tags; i++) + match_tags[i] = reach_next_i->tags[i]; + } + reach_pos[trans_i->state_id].pos = pos; + reach_pos[trans_i->state_id].tags = &reach_next_i->tags; + reach_next_i++; + } + trans_i++; + } + reach_next_i->state = NULL; + } + else + { + if (num_tags == 0 || reach_next_i == reach_next) + /* We have found a match. */ + break; + } + + /* Check for end of string. */ + if (!next_c) break; + + GET_NEXT_WCHAR(); + + /* Swap `reach' and `reach_next'. */ + reach_i = reach; + reach = reach_next; + reach_next = reach_i; + + /* For each state in `reach', weed out states that don't fulfill the + minimal matching conditions. */ + if (tnfa->num_minimals && new_match) + { + new_match = 0; + reach_next_i = reach_next; + for (reach_i = reach; reach_i->state; reach_i++) + { + int skip = 0; + for (i = 0; tnfa->minimal_tags[i] >= 0; i += 2) + { + int end = tnfa->minimal_tags[i]; + int start = tnfa->minimal_tags[i + 1]; + if (end >= num_tags) + { + skip = 1; + break; + } + else if (reach_i->tags[start] == match_tags[start] + && reach_i->tags[end] < match_tags[end]) + { + skip = 1; + break; + } + } + if (!skip) + { + reach_next_i->state = reach_i->state; + tmp_iptr = reach_next_i->tags; + reach_next_i->tags = reach_i->tags; + reach_i->tags = tmp_iptr; + reach_next_i++; + } + } + reach_next_i->state = NULL; + + /* Swap `reach' and `reach_next'. */ + reach_i = reach; + reach = reach_next; + reach_next = reach_i; + } + + /* For each state in `reach' see if there is a transition leaving with + the current input symbol to a state not yet in `reach_next', and + add the destination states to `reach_next'. */ + reach_next_i = reach_next; + for (reach_i = reach; reach_i->state; reach_i++) + { + for (trans_i = reach_i->state; trans_i->state; trans_i++) + { + /* Does this transition match the input symbol? */ + if (trans_i->code_min <= (tre_cint_t)prev_c && + trans_i->code_max >= (tre_cint_t)prev_c) + { + if (trans_i->assertions + && (CHECK_ASSERTIONS(trans_i->assertions) + || CHECK_CHAR_CLASSES(trans_i, tnfa, eflags))) + { + continue; + } + + /* Compute the tags after this transition. */ + for (i = 0; i < num_tags; i++) + tmp_tags[i] = reach_i->tags[i]; + tag_i = trans_i->tags; + if (tag_i != NULL) + while (*tag_i >= 0) + { + if (*tag_i < num_tags) + tmp_tags[*tag_i] = pos; + tag_i++; + } + + if (reach_pos[trans_i->state_id].pos < pos) + { + /* Found an unvisited node. */ + reach_next_i->state = trans_i->state; + tmp_iptr = reach_next_i->tags; + reach_next_i->tags = tmp_tags; + tmp_tags = tmp_iptr; + reach_pos[trans_i->state_id].pos = pos; + reach_pos[trans_i->state_id].tags = &reach_next_i->tags; + + if (reach_next_i->state == tnfa->final + && (match_eo == -1 + || (num_tags > 0 + && reach_next_i->tags[0] <= match_tags[0]))) + { + match_eo = pos; + new_match = 1; + for (i = 0; i < num_tags; i++) + match_tags[i] = reach_next_i->tags[i]; + } + reach_next_i++; + + } + else + { + assert(reach_pos[trans_i->state_id].pos == pos); + /* Another path has also reached this state. We choose + the winner by examining the tag values for both + paths. */ + if (tre_tag_order(num_tags, tnfa->tag_directions, + tmp_tags, + *reach_pos[trans_i->state_id].tags)) + { + /* The new path wins. */ + tmp_iptr = *reach_pos[trans_i->state_id].tags; + *reach_pos[trans_i->state_id].tags = tmp_tags; + if (trans_i->state == tnfa->final) + { + match_eo = pos; + new_match = 1; + for (i = 0; i < num_tags; i++) + match_tags[i] = tmp_tags[i]; + } + tmp_tags = tmp_iptr; + } + } + } + } + } + reach_next_i->state = NULL; + } + + *match_end_ofs = match_eo; + ret = match_eo >= 0 ? REG_OK : REG_NOMATCH; +error_exit: + xfree(buf); + return ret; +} + + + +/*********************************************************************** + from tre-match-backtrack.c +***********************************************************************/ + +/* + This matcher is for regexps that use back referencing. Regexp matching + with back referencing is an NP-complete problem on the number of back + references. The easiest way to match them is to use a backtracking + routine which basically goes through all possible paths in the TNFA + and chooses the one which results in the best (leftmost and longest) + match. This can be spectacularly expensive and may run out of stack + space, but there really is no better known generic algorithm. Quoting + Henry Spencer from comp.compilers: + <URL: http://compilers.iecc.com/comparch/article/93-03-102> + + POSIX.2 REs require longest match, which is really exciting to + implement since the obsolete ("basic") variant also includes + \<digit>. I haven't found a better way of tackling this than doing + a preliminary match using a DFA (or simulation) on a modified RE + that just replicates subREs for \<digit>, and then doing a + backtracking match to determine whether the subRE matches were + right. This can be rather slow, but I console myself with the + thought that people who use \<digit> deserve very slow execution. + (Pun unintentional but very appropriate.) + +*/ + +typedef struct { + regoff_t pos; + const char *str_byte; + tre_tnfa_transition_t *state; + int state_id; + int next_c; + regoff_t *tags; +#ifdef TRE_MBSTATE + mbstate_t mbstate; +#endif /* TRE_MBSTATE */ +} tre_backtrack_item_t; + +typedef struct tre_backtrack_struct { + tre_backtrack_item_t item; + struct tre_backtrack_struct *prev; + struct tre_backtrack_struct *next; +} *tre_backtrack_t; + +#ifdef TRE_MBSTATE +#define BT_STACK_MBSTATE_IN stack->item.mbstate = (mbstate) +#define BT_STACK_MBSTATE_OUT (mbstate) = stack->item.mbstate +#else /* !TRE_MBSTATE */ +#define BT_STACK_MBSTATE_IN +#define BT_STACK_MBSTATE_OUT +#endif /* !TRE_MBSTATE */ + +#define tre_bt_mem_new tre_mem_new +#define tre_bt_mem_alloc tre_mem_alloc +#define tre_bt_mem_destroy tre_mem_destroy + + +#define BT_STACK_PUSH(_pos, _str_byte, _str_wide, _state, _state_id, _next_c, _tags, _mbstate) \ + do \ + { \ + int i; \ + if (!stack->next) \ + { \ + tre_backtrack_t s; \ + s = tre_bt_mem_alloc(mem, sizeof(*s)); \ + if (!s) \ + { \ + tre_bt_mem_destroy(mem); \ + if (tags) \ + xfree(tags); \ + if (pmatch) \ + xfree(pmatch); \ + if (states_seen) \ + xfree(states_seen); \ + return REG_ESPACE; \ + } \ + s->prev = stack; \ + s->next = NULL; \ + s->item.tags = tre_bt_mem_alloc(mem, \ + sizeof(*tags) * tnfa->num_tags); \ + if (!s->item.tags) \ + { \ + tre_bt_mem_destroy(mem); \ + if (tags) \ + xfree(tags); \ + if (pmatch) \ + xfree(pmatch); \ + if (states_seen) \ + xfree(states_seen); \ + return REG_ESPACE; \ + } \ + stack->next = s; \ + stack = s; \ + } \ + else \ + stack = stack->next; \ + stack->item.pos = (_pos); \ + stack->item.str_byte = (_str_byte); \ + stack->item.state = (_state); \ + stack->item.state_id = (_state_id); \ + stack->item.next_c = (_next_c); \ + for (i = 0; i < tnfa->num_tags; i++) \ + stack->item.tags[i] = (_tags)[i]; \ + BT_STACK_MBSTATE_IN; \ + } \ + while (0) + +#define BT_STACK_POP() \ + do \ + { \ + int i; \ + assert(stack->prev); \ + pos = stack->item.pos; \ + str_byte = stack->item.str_byte; \ + state = stack->item.state; \ + next_c = stack->item.next_c; \ + for (i = 0; i < tnfa->num_tags; i++) \ + tags[i] = stack->item.tags[i]; \ + BT_STACK_MBSTATE_OUT; \ + stack = stack->prev; \ + } \ + while (0) + +#undef MIN +#define MIN(a, b) ((a) <= (b) ? (a) : (b)) + +static reg_errcode_t +tre_tnfa_run_backtrack(const tre_tnfa_t *tnfa, const void *string, + regoff_t *match_tags, int eflags, regoff_t *match_end_ofs) +{ + /* State variables required by GET_NEXT_WCHAR. */ + tre_char_t prev_c = 0, next_c = 0; + const char *str_byte = string; + regoff_t pos = 0; + regoff_t pos_add_next = 1; +#ifdef TRE_MBSTATE + mbstate_t mbstate; +#endif /* TRE_MBSTATE */ + int reg_notbol = eflags & REG_NOTBOL; + int reg_noteol = eflags & REG_NOTEOL; + int reg_newline = tnfa->cflags & REG_NEWLINE; + + /* These are used to remember the necessary values of the above + variables to return to the position where the current search + started from. */ + int next_c_start; + const char *str_byte_start; + regoff_t pos_start = -1; +#ifdef TRE_MBSTATE + mbstate_t mbstate_start; +#endif /* TRE_MBSTATE */ + + /* End offset of best match so far, or -1 if no match found yet. */ + regoff_t match_eo = -1; + /* Tag arrays. */ + int *next_tags; + regoff_t *tags = NULL; + /* Current TNFA state. */ + tre_tnfa_transition_t *state; + int *states_seen = NULL; + + /* Memory allocator to for allocating the backtracking stack. */ + tre_mem_t mem = tre_bt_mem_new(); + + /* The backtracking stack. */ + tre_backtrack_t stack; + + tre_tnfa_transition_t *trans_i; + regmatch_t *pmatch = NULL; + int ret; + +#ifdef TRE_MBSTATE + memset(&mbstate, '\0', sizeof(mbstate)); +#endif /* TRE_MBSTATE */ + + if (!mem) + return REG_ESPACE; + stack = tre_bt_mem_alloc(mem, sizeof(*stack)); + if (!stack) + { + ret = REG_ESPACE; + goto error_exit; + } + stack->prev = NULL; + stack->next = NULL; + + if (tnfa->num_tags) + { + tags = xmalloc(sizeof(*tags) * tnfa->num_tags); + if (!tags) + { + ret = REG_ESPACE; + goto error_exit; + } + } + if (tnfa->num_submatches) + { + pmatch = xmalloc(sizeof(*pmatch) * tnfa->num_submatches); + if (!pmatch) + { + ret = REG_ESPACE; + goto error_exit; + } + } + if (tnfa->num_states) + { + states_seen = xmalloc(sizeof(*states_seen) * tnfa->num_states); + if (!states_seen) + { + ret = REG_ESPACE; + goto error_exit; + } + } + + retry: + { + int i; + for (i = 0; i < tnfa->num_tags; i++) + { + tags[i] = -1; + if (match_tags) + match_tags[i] = -1; + } + for (i = 0; i < tnfa->num_states; i++) + states_seen[i] = 0; + } + + state = NULL; + pos = pos_start; + GET_NEXT_WCHAR(); + pos_start = pos; + next_c_start = next_c; + str_byte_start = str_byte; +#ifdef TRE_MBSTATE + mbstate_start = mbstate; +#endif /* TRE_MBSTATE */ + + /* Handle initial states. */ + next_tags = NULL; + for (trans_i = tnfa->initial; trans_i->state; trans_i++) + { + if (trans_i->assertions && CHECK_ASSERTIONS(trans_i->assertions)) + { + continue; + } + if (state == NULL) + { + /* Start from this state. */ + state = trans_i->state; + next_tags = trans_i->tags; + } + else + { + /* Backtrack to this state. */ + BT_STACK_PUSH(pos, str_byte, 0, trans_i->state, + trans_i->state_id, next_c, tags, mbstate); + { + int *tmp = trans_i->tags; + if (tmp) + while (*tmp >= 0) + stack->item.tags[*tmp++] = pos; + } + } + } + + if (next_tags) + for (; *next_tags >= 0; next_tags++) + tags[*next_tags] = pos; + + + if (state == NULL) + goto backtrack; + + while (1) + { + tre_tnfa_transition_t *next_state; + int empty_br_match; + + if (state == tnfa->final) + { + if (match_eo < pos + || (match_eo == pos + && match_tags + && tre_tag_order(tnfa->num_tags, tnfa->tag_directions, + tags, match_tags))) + { + int i; + /* This match wins the previous match. */ + match_eo = pos; + if (match_tags) + for (i = 0; i < tnfa->num_tags; i++) + match_tags[i] = tags[i]; + } + /* Our TNFAs never have transitions leaving from the final state, + so we jump right to backtracking. */ + goto backtrack; + } + + /* Go to the next character in the input string. */ + empty_br_match = 0; + trans_i = state; + if (trans_i->state && trans_i->assertions & ASSERT_BACKREF) + { + /* This is a back reference state. All transitions leaving from + this state have the same back reference "assertion". Instead + of reading the next character, we match the back reference. */ + regoff_t so, eo; + int bt = trans_i->u.backref; + regoff_t bt_len; + int result; + + /* Get the substring we need to match against. Remember to + turn off REG_NOSUB temporarily. */ + tre_fill_pmatch(bt + 1, pmatch, tnfa->cflags & ~REG_NOSUB, + tnfa, tags, pos); + so = pmatch[bt].rm_so; + eo = pmatch[bt].rm_eo; + bt_len = eo - so; + + result = strncmp((const char*)string + so, str_byte - 1, + (size_t)bt_len); + + if (result == 0) + { + /* Back reference matched. Check for infinite loop. */ + if (bt_len == 0) + empty_br_match = 1; + if (empty_br_match && states_seen[trans_i->state_id]) + { + goto backtrack; + } + + states_seen[trans_i->state_id] = empty_br_match; + + /* Advance in input string and resync `prev_c', `next_c' + and pos. */ + str_byte += bt_len - 1; + pos += bt_len - 1; + GET_NEXT_WCHAR(); + } + else + { + goto backtrack; + } + } + else + { + /* Check for end of string. */ + if (next_c == L'\0') + goto backtrack; + + /* Read the next character. */ + GET_NEXT_WCHAR(); + } + + next_state = NULL; + for (trans_i = state; trans_i->state; trans_i++) + { + if (trans_i->code_min <= (tre_cint_t)prev_c + && trans_i->code_max >= (tre_cint_t)prev_c) + { + if (trans_i->assertions + && (CHECK_ASSERTIONS(trans_i->assertions) + || CHECK_CHAR_CLASSES(trans_i, tnfa, eflags))) + { + continue; + } + + if (next_state == NULL) + { + /* First matching transition. */ + next_state = trans_i->state; + next_tags = trans_i->tags; + } + else + { + /* Second matching transition. We may need to backtrack here + to take this transition instead of the first one, so we + push this transition in the backtracking stack so we can + jump back here if needed. */ + BT_STACK_PUSH(pos, str_byte, 0, trans_i->state, + trans_i->state_id, next_c, tags, mbstate); + { + int *tmp; + for (tmp = trans_i->tags; tmp && *tmp >= 0; tmp++) + stack->item.tags[*tmp] = pos; + } +#if 0 /* XXX - it's important not to look at all transitions here to keep + the stack small! */ + break; +#endif + } + } + } + + if (next_state != NULL) + { + /* Matching transitions were found. Take the first one. */ + state = next_state; + + /* Update the tag values. */ + if (next_tags) + while (*next_tags >= 0) + tags[*next_tags++] = pos; + } + else + { + backtrack: + /* A matching transition was not found. Try to backtrack. */ + if (stack->prev) + { + if (stack->item.state->assertions & ASSERT_BACKREF) + { + states_seen[stack->item.state_id] = 0; + } + + BT_STACK_POP(); + } + else if (match_eo < 0) + { + /* Try starting from a later position in the input string. */ + /* Check for end of string. */ + if (next_c == L'\0') + { + break; + } + next_c = next_c_start; +#ifdef TRE_MBSTATE + mbstate = mbstate_start; +#endif /* TRE_MBSTATE */ + str_byte = str_byte_start; + goto retry; + } + else + { + break; + } + } + } + + ret = match_eo >= 0 ? REG_OK : REG_NOMATCH; + *match_end_ofs = match_eo; + + error_exit: + tre_bt_mem_destroy(mem); +#ifndef TRE_USE_ALLOCA + if (tags) + xfree(tags); + if (pmatch) + xfree(pmatch); + if (states_seen) + xfree(states_seen); +#endif /* !TRE_USE_ALLOCA */ + + return ret; +} + +/*********************************************************************** + from regexec.c +***********************************************************************/ + +/* Fills the POSIX.2 regmatch_t array according to the TNFA tag and match + endpoint values. */ +static void +tre_fill_pmatch(size_t nmatch, regmatch_t pmatch[], int cflags, + const tre_tnfa_t *tnfa, regoff_t *tags, regoff_t match_eo) +{ + tre_submatch_data_t *submatch_data; + unsigned int i, j; + int *parents; + + i = 0; + if (match_eo >= 0 && !(cflags & REG_NOSUB)) + { + /* Construct submatch offsets from the tags. */ + submatch_data = tnfa->submatch_data; + while (i < tnfa->num_submatches && i < nmatch) + { + if (submatch_data[i].so_tag == tnfa->end_tag) + pmatch[i].rm_so = match_eo; + else + pmatch[i].rm_so = tags[submatch_data[i].so_tag]; + + if (submatch_data[i].eo_tag == tnfa->end_tag) + pmatch[i].rm_eo = match_eo; + else + pmatch[i].rm_eo = tags[submatch_data[i].eo_tag]; + + /* If either of the endpoints were not used, this submatch + was not part of the match. */ + if (pmatch[i].rm_so == -1 || pmatch[i].rm_eo == -1) + pmatch[i].rm_so = pmatch[i].rm_eo = -1; + + i++; + } + /* Reset all submatches that are not within all of their parent + submatches. */ + i = 0; + while (i < tnfa->num_submatches && i < nmatch) + { + if (pmatch[i].rm_eo == -1) + assert(pmatch[i].rm_so == -1); + assert(pmatch[i].rm_so <= pmatch[i].rm_eo); + + parents = submatch_data[i].parents; + if (parents != NULL) + for (j = 0; parents[j] >= 0; j++) + { + if (pmatch[i].rm_so < pmatch[parents[j]].rm_so + || pmatch[i].rm_eo > pmatch[parents[j]].rm_eo) + pmatch[i].rm_so = pmatch[i].rm_eo = -1; + } + i++; + } + } + + while (i < nmatch) + { + pmatch[i].rm_so = -1; + pmatch[i].rm_eo = -1; + i++; + } +} + + +/* + Wrapper functions for POSIX compatible regexp matching. +*/ + +int +regexec(const regex_t *restrict preg, const char *restrict string, + size_t nmatch, regmatch_t pmatch[restrict], int eflags) +{ + tre_tnfa_t *tnfa = (void *)preg->TRE_REGEX_T_FIELD; + reg_errcode_t status; + regoff_t *tags = NULL, eo; + if (tnfa->cflags & REG_NOSUB) nmatch = 0; + if (tnfa->num_tags > 0 && nmatch > 0) + { + tags = xmalloc(sizeof(*tags) * tnfa->num_tags); + if (tags == NULL) + return REG_ESPACE; + } + + /* Dispatch to the appropriate matcher. */ + if (tnfa->have_backrefs) + { + /* The regex has back references, use the backtracking matcher. */ + status = tre_tnfa_run_backtrack(tnfa, string, tags, eflags, &eo); + } + else + { + /* Exact matching, no back references, use the parallel matcher. */ + status = tre_tnfa_run_parallel(tnfa, string, tags, eflags, &eo); + } + + if (status == REG_OK) + /* A match was found, so fill the submatch registers. */ + tre_fill_pmatch(nmatch, pmatch, tnfa->cflags, tnfa, tags, eo); + if (tags) + xfree(tags); + return status; +} diff --git a/vendor/tre-regex/tre-mem.c b/vendor/tre-regex/tre-mem.c new file mode 100644 index 0000000..86f809d --- /dev/null +++ b/vendor/tre-regex/tre-mem.c @@ -0,0 +1,158 @@ +/* + tre-mem.c - TRE memory allocator + + Copyright (c) 2001-2009 Ville Laurikari <vl@iki.fi> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +/* + This memory allocator is for allocating small memory blocks efficiently + in terms of memory overhead and execution speed. The allocated blocks + cannot be freed individually, only all at once. There can be multiple + allocators, though. +*/ + +#include <stdlib.h> +#include <string.h> + +#include "tre.h" + +/* + This memory allocator is for allocating small memory blocks efficiently + in terms of memory overhead and execution speed. The allocated blocks + cannot be freed individually, only all at once. There can be multiple + allocators, though. +*/ + +/* Returns a new memory allocator or NULL if out of memory. */ +tre_mem_t +tre_mem_new_impl(int provided, void *provided_block) +{ + tre_mem_t mem; + if (provided) + { + mem = provided_block; + memset(mem, 0, sizeof(*mem)); + } + else + mem = xcalloc(1, sizeof(*mem)); + if (mem == NULL) + return NULL; + return mem; +} + + +/* Frees the memory allocator and all memory allocated with it. */ +void +tre_mem_destroy(tre_mem_t mem) +{ + tre_list_t *tmp, *l = mem->blocks; + + while (l != NULL) + { + xfree(l->data); + tmp = l->next; + xfree(l); + l = tmp; + } + xfree(mem); +} + + +/* Allocates a block of `size' bytes from `mem'. Returns a pointer to the + allocated block or NULL if an underlying malloc() failed. */ +void * +tre_mem_alloc_impl(tre_mem_t mem, int provided, void *provided_block, + int zero, size_t size) +{ + void *ptr; + + if (mem->failed) + { + return NULL; + } + + if (mem->n < size) + { + /* We need more memory than is available in the current block. + Allocate a new block. */ + tre_list_t *l; + if (provided) + { + if (provided_block == NULL) + { + mem->failed = 1; + return NULL; + } + mem->ptr = provided_block; + mem->n = TRE_MEM_BLOCK_SIZE; + } + else + { + int block_size; + if (size * 8 > TRE_MEM_BLOCK_SIZE) + block_size = size * 8; + else + block_size = TRE_MEM_BLOCK_SIZE; + l = xmalloc(sizeof(*l)); + if (l == NULL) + { + mem->failed = 1; + return NULL; + } + l->data = xmalloc(block_size); + if (l->data == NULL) + { + xfree(l); + mem->failed = 1; + return NULL; + } + l->next = NULL; + if (mem->current != NULL) + mem->current->next = l; + if (mem->blocks == NULL) + mem->blocks = l; + mem->current = l; + mem->ptr = l->data; + mem->n = block_size; + } + } + + /* Make sure the next pointer will be aligned. */ + size += ALIGN(mem->ptr + size, long); + + /* Allocate from current block. */ + ptr = mem->ptr; + mem->ptr += size; + mem->n -= size; + + /* Set to zero if needed. */ + if (zero) + memset(ptr, 0, size); + + return ptr; +} diff --git a/vendor/tre-regex/tre.h b/vendor/tre-regex/tre.h new file mode 100644 index 0000000..923e2d5 --- /dev/null +++ b/vendor/tre-regex/tre.h @@ -0,0 +1,267 @@ +/* + tre-internal.h - TRE internal definitions + + Copyright (c) 2001-2009 Ville Laurikari <vl@iki.fi> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "regex.h" +#include <stdint.h> +#include <wchar.h> +#include <wctype.h> + +#define hidden + +#undef TRE_MBSTATE + +#ifndef NDEBUG +#define NDEBUG +#endif + +#ifndef CHARCLASS_NAME_MAX +#define CHARCLASS_NAME_MAX 14 +#endif +#ifndef RE_DUP_MAX +#define RE_DUP_MAX 255 +#endif + +#define TRE_REGEX_T_FIELD __opaque +typedef int reg_errcode_t; + +typedef int tre_char_t; + +#define DPRINT(msg) do { } while(0) + +#define elementsof(x) ( sizeof(x) / sizeof(x[0]) ) + +/* wchar_t is 16 bits on Windows: decode UTF-8 to code points ourselves. + Returns the sequence length, 0 at NUL, -1 on invalid input. */ +static inline int tre_mbtowc(tre_char_t *pwc, const char *s, size_t n) +{ + const unsigned char *p = (const unsigned char *)s; + int c, len, i; + if (n == 0) return -1; + c = p[0]; + if (c < 0x80) { *pwc = c; return c ? 1 : 0; } + if (c < 0xC2) return -1; + if (c < 0xE0) { len = 2; c &= 0x1F; } + else if (c < 0xF0) { len = 3; c &= 0x0F; } + else if (c < 0xF5) { len = 4; c &= 0x07; } + else return -1; + if ((size_t)len > n) return -1; + for (i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) return -1; + c = (c << 6) | (p[i] & 0x3F); + } + if ((len == 3 && c < 0x800) || (len == 4 && (c < 0x10000 || c > 0x10FFFF)) + || (c >= 0xD800 && c <= 0xDFFF)) return -1; + *pwc = c; + return len; +} +#define tre_mbrtowc(pwc, s, n, ps) (tre_mbtowc((pwc), (s), (n))) + +/* Wide characters. */ +typedef int tre_cint_t; +#define TRE_CHAR_MAX 0x10ffff + +#define tre_isalnum(c) ((c) <= 0xFFFF ? iswalnum((wint_t)(c)) : 1) +#define tre_isalpha(c) ((c) <= 0xFFFF ? iswalpha((wint_t)(c)) : 1) +#define tre_isblank(c) ((c) <= 0xFFFF ? iswblank((wint_t)(c)) : 0) +#define tre_iscntrl(c) ((c) <= 0xFFFF ? iswcntrl((wint_t)(c)) : 0) +#define tre_isdigit(c) ((c) <= 0xFFFF ? iswdigit((wint_t)(c)) : 0) +#define tre_isgraph(c) ((c) <= 0xFFFF ? iswgraph((wint_t)(c)) : 1) +#define tre_islower(c) ((c) <= 0xFFFF ? iswlower((wint_t)(c)) : 0) +#define tre_isprint(c) ((c) <= 0xFFFF ? iswprint((wint_t)(c)) : 1) +#define tre_ispunct(c) ((c) <= 0xFFFF ? iswpunct((wint_t)(c)) : 0) +#define tre_isspace(c) ((c) <= 0xFFFF ? iswspace((wint_t)(c)) : 0) +#define tre_isupper(c) ((c) <= 0xFFFF ? iswupper((wint_t)(c)) : 0) +#define tre_isxdigit(c) ((c) <= 0xFFFF ? iswxdigit((wint_t)(c)) : 0) + +#define tre_tolower(c) ((c) <= 0xFFFF ? (int)towlower((wint_t)(c)) : (c)) +#define tre_toupper(c) ((c) <= 0xFFFF ? (int)towupper((wint_t)(c)) : (c)) +#define tre_strlen wcslen + +/* Use system provided iswctype() and wctype(). */ +typedef wctype_t tre_ctype_t; +#define tre_isctype(c, t) ((c) <= 0xFFFF ? iswctype((wint_t)(c), (t)) : 0) +#define tre_ctype wctype + +/* Returns number of bytes to add to (char *)ptr to make it + properly aligned for the type. */ +#define ALIGN(ptr, type) \ + ((((uintptr_t)ptr) % sizeof(type)) \ + ? (sizeof(type) - (((uintptr_t)ptr) % sizeof(type))) \ + : 0) + +#undef MAX +#undef MIN +#define MAX(a, b) (((a) >= (b)) ? (a) : (b)) +#define MIN(a, b) (((a) <= (b)) ? (a) : (b)) + +/* TNFA transition type. A TNFA state is an array of transitions, + the terminator is a transition with NULL `state'. */ +typedef struct tnfa_transition tre_tnfa_transition_t; + +struct tnfa_transition { + /* Range of accepted characters. */ + tre_cint_t code_min; + tre_cint_t code_max; + /* Pointer to the destination state. */ + tre_tnfa_transition_t *state; + /* ID number of the destination state. */ + int state_id; + /* -1 terminated array of tags (or NULL). */ + int *tags; + /* Assertion bitmap. */ + int assertions; + /* Assertion parameters. */ + union { + /* Character class assertion. */ + tre_ctype_t class; + /* Back reference assertion. */ + int backref; + } u; + /* Negative character class assertions. */ + tre_ctype_t *neg_classes; +}; + + +/* Assertions. */ +#define ASSERT_AT_BOL 1 /* Beginning of line. */ +#define ASSERT_AT_EOL 2 /* End of line. */ +#define ASSERT_CHAR_CLASS 4 /* Character class in `class'. */ +#define ASSERT_CHAR_CLASS_NEG 8 /* Character classes in `neg_classes'. */ +#define ASSERT_AT_BOW 16 /* Beginning of word. */ +#define ASSERT_AT_EOW 32 /* End of word. */ +#define ASSERT_AT_WB 64 /* Word boundary. */ +#define ASSERT_AT_WB_NEG 128 /* Not a word boundary. */ +#define ASSERT_BACKREF 256 /* A back reference in `backref'. */ +#define ASSERT_LAST 256 + +/* Tag directions. */ +typedef enum { + TRE_TAG_MINIMIZE = 0, + TRE_TAG_MAXIMIZE = 1 +} tre_tag_direction_t; + +/* Instructions to compute submatch register values from tag values + after a successful match. */ +struct tre_submatch_data { + /* Tag that gives the value for rm_so (submatch start offset). */ + int so_tag; + /* Tag that gives the value for rm_eo (submatch end offset). */ + int eo_tag; + /* List of submatches this submatch is contained in. */ + int *parents; +}; + +typedef struct tre_submatch_data tre_submatch_data_t; + + +/* TNFA definition. */ +typedef struct tnfa tre_tnfa_t; + +struct tnfa { + tre_tnfa_transition_t *transitions; + unsigned int num_transitions; + tre_tnfa_transition_t *initial; + tre_tnfa_transition_t *final; + tre_submatch_data_t *submatch_data; + char *firstpos_chars; + int first_char; + unsigned int num_submatches; + tre_tag_direction_t *tag_directions; + int *minimal_tags; + int num_tags; + int num_minimals; + int end_tag; + int num_states; + int cflags; + int have_backrefs; + int have_approx; +}; + +/* from tre-mem.h: */ + +#define TRE_MEM_BLOCK_SIZE 1024 + +typedef struct tre_list { + void *data; + struct tre_list *next; +} tre_list_t; + +typedef struct tre_mem_struct { + tre_list_t *blocks; + tre_list_t *current; + char *ptr; + size_t n; + int failed; + void **provided; +} *tre_mem_t; + +#define tre_mem_new_impl __tre_mem_new_impl +#define tre_mem_alloc_impl __tre_mem_alloc_impl +#define tre_mem_destroy __tre_mem_destroy + +hidden tre_mem_t tre_mem_new_impl(int provided, void *provided_block); +hidden void *tre_mem_alloc_impl(tre_mem_t mem, int provided, void *provided_block, + int zero, size_t size); + +/* Returns a new memory allocator or NULL if out of memory. */ +#define tre_mem_new() tre_mem_new_impl(0, NULL) + +/* Allocates a block of `size' bytes from `mem'. Returns a pointer to the + allocated block or NULL if an underlying malloc() failed. */ +#define tre_mem_alloc(mem, size) tre_mem_alloc_impl(mem, 0, NULL, 0, size) + +/* Allocates a block of `size' bytes from `mem'. Returns a pointer to the + allocated block or NULL if an underlying malloc() failed. The memory + is set to zero. */ +#define tre_mem_calloc(mem, size) tre_mem_alloc_impl(mem, 0, NULL, 1, size) + +#ifdef TRE_USE_ALLOCA +/* alloca() versions. Like above, but memory is allocated with alloca() + instead of malloc(). */ + +#define tre_mem_newa() \ + tre_mem_new_impl(1, alloca(sizeof(struct tre_mem_struct))) + +#define tre_mem_alloca(mem, size) \ + ((mem)->n >= (size) \ + ? tre_mem_alloc_impl((mem), 1, NULL, 0, (size)) \ + : tre_mem_alloc_impl((mem), 1, alloca(TRE_MEM_BLOCK_SIZE), 0, (size))) +#endif /* TRE_USE_ALLOCA */ + + +/* Frees the memory allocator and all memory allocated with it. */ +hidden void tre_mem_destroy(tre_mem_t mem); + +#define xmalloc malloc +#define xcalloc calloc +#define xfree free +#define xrealloc realloc +