From c2320f946b67519640ae64fe4e130ba351293f03 Mon Sep 17 00:00:00 2001 From: WHC Date: Sun, 11 Oct 2026 15:22:10 +0800 Subject: [PATCH] v0.9.0: the Qt front end as an Android APK, images in the transcript Two strands land together: the window draws pictures now, and the whole front end builds, installs and runs on the phone with the engine linked in-process. Images (the "make a picture actually show up" part) - Engine, additively: cst_agent_hooks_t.on_tool_images(call, list, n, ud) fires when a call left attachments behind, and cst_agent_pending_rel() / cst_agent_pending_is_image() read that list without the app touching the store. tests/test_attach.c::test_agent_pending pins the three cases (an image, a stored name that passes cst_attach_rel_ok, out-of-range NULL). - App: image://cstattach/ (app/src/ui/SessionImages.cpp) serves only the current session's attachment directory, checked with cst_attach_rel_ok, decodes down to the requested sourceSize and never upscales, and holds a mutex because requestImage() runs on the render thread while the directory can change underneath it. AgentCore gained attachmentDir(), pendingImages(), the toolImages()/promptImages() signals and hookToolImages(); history() gives replayed rows their images too. - QML: ImageStrip.qml (a thumb strip, <=150 px tall, shaped by the picture, tooltip with name/size/origin, the file's name when it went missing), wired into MessageCard and ToolCard. Qt 6's ListModel cannot hold a list-valued role (append() reads back undefined, setProperty drops it), so a row carries one image per line as |||x and ImageStrip decodes that. Android (the engine is already in the APK; this is the front end) - app/src/main.cpp: on Android the workspace, config, tmp and HOME go under the app's own data directory before the engine opens anything, because QDir::currentPath() is "/" there and /tmp is not writable. - app/android/: the manifest (QtApplication/QtActivity, INTERNET and POST_NOTIFICATIONS, adjustResize, no storage permission -- the picker will be the system one), strings, a vector icon; minSdk 24 and targetSdk 35 come from app/CMakeLists.txt through QT_ANDROID_MIN/TARGET_SDK_VERSION. - app/src/platform/android/AndroidTools.cpp: the Android file behind PlatformInit(), owner "app:android" (notification, clipboard, share sheet, file picker are stubs for now, so the tool list does not lie about them). - app/tools/selftest.c decides its scratch workspace with os_temp_dir(), so the C smoke test runs on the device too. - scripts/build_apk_android.sh: sources to installed APK in one command (environment checks, qt-cmake configure, --target apk, adb install), and the host-side plumbing the NixOS + qt.io combination needs -- the NDK sysroot include dirs for Qt's FindGLESv2/FindEGL (they only look under /include), -L$PREFIX/lib on the shared/module link lines as well as the executables', the Android debug keystore created on first use with QT_ANDROID_SIGN_APK + QT_ANDROID_KEYSTORE_* (androiddeployqt copies the unsigned gradle output and signs it itself), and the #!/bin/bash shebang of the build-tools wrappers, which NixOS does not have. - scripts/gradle-android-init.gradle (installed into ~/.gradle/init.d/): the release lint task is switched off -- lintVitalAnalyzeRelease resolves its own tool jars straight from dl.google.com, bypassing every repository the build configures, and fails the build after all the real work is done. - Icons: app/icons/*.svg (Breeze's 24@3x files, the 19 names the window asks for) are compiled in as :/cst-icons and ThemeIcons.cpp falls back to them when the platform has no icon theme. On the phone the Qt theme lookup found nothing and the QML glyph fallback was an empty box in the phone's fonts. - docs/android.md: "The APK (Qt front end)" -- the SDK assembled without sdkmanager, the host Qt's missing libraries on NixOS, the Gradle mirrors, lint, signing -- and a Status that says what runs where. app/README.md gains the images and APK lines, and stops claiming Android platform tools that are still stubs. Verified: engine `make check` green; desktop `cst-app-selftest: OK` and --uitest all checks pass; phone (NX789J, Android 16, arm64-v8a) -- the APK installs, starts on the workspace under /data/user/0/xyz.whcself.cstagent/ files/, a turn reaches a configured provider over HTTPS from inside the ART process and renders the reply, rotation keeps the session and docks the sidebar, and the toolbar/card icons draw. Version 0.9.0. mode = coding --- .gitignore | 8 ++ app/CMakeLists.txt | 80 ++++++++++- app/README.md | 31 +++- app/android/AndroidManifest.xml | 73 ++++++++++ app/android/res/drawable/ic_launcher.xml | 30 ++++ app/android/res/values/strings.xml | 5 + app/icons/application-menu.svg | 9 ++ app/icons/arrow-down.svg | 9 ++ app/icons/arrow-right.svg | 9 ++ app/icons/box.svg | 10 ++ app/icons/code-context.svg | 9 ++ app/icons/document-edit.svg | 9 ++ app/icons/edit-copy.svg | 10 ++ app/icons/edit-delete.svg | 9 ++ app/icons/edit-find.svg | 9 ++ app/icons/folder-open.svg | 9 ++ app/icons/format-list-unordered.svg | 9 ++ app/icons/go-down.svg | 9 ++ app/icons/help-hint.svg | 9 ++ app/icons/internet-services.svg | 11 ++ app/icons/list-add.svg | 9 ++ app/icons/mail-attachment.svg | 9 ++ app/icons/settings-configure.svg | 9 ++ app/icons/task-complete.svg | 9 ++ app/icons/text-x-generic.svg | 14 ++ app/qml/ChatView.qml | 2 + app/qml/ImageStrip.qml | 110 ++++++++++++++ app/qml/Main.qml | 35 ++++- app/qml/MessageCard.qml | 8 ++ app/qml/ToolCard.qml | 9 ++ app/src/core/AgentCore.cpp | 100 ++++++++++++- app/src/core/AgentCore.h | 21 ++- app/src/main.cpp | 40 +++++- app/src/platform/android/AndroidTools.cpp | 43 ++++++ app/src/ui/SessionImages.cpp | 81 +++++++++++ app/src/ui/SessionImages.h | 45 ++++++ app/src/ui/ThemeIcons.cpp | 21 ++- app/src/ui/UiTest.cpp | 125 ++++++++++++++++ app/tools/selftest.c | 11 +- docs/android.md | 104 +++++++++++++- include/cst_agent.h | 12 ++ include/cst_version.h | 4 +- scripts/build_apk_android.sh | 168 ++++++++++++++++++++++ scripts/gradle-android-init.gradle | 25 ++++ src/agent_loop.c | 16 +++ tests/test_attach.c | 9 ++ 46 files changed, 1370 insertions(+), 26 deletions(-) create mode 100644 app/android/AndroidManifest.xml create mode 100644 app/android/res/drawable/ic_launcher.xml create mode 100644 app/android/res/values/strings.xml create mode 100644 app/icons/application-menu.svg create mode 100644 app/icons/arrow-down.svg create mode 100644 app/icons/arrow-right.svg create mode 100644 app/icons/box.svg create mode 100644 app/icons/code-context.svg create mode 100644 app/icons/document-edit.svg create mode 100644 app/icons/edit-copy.svg create mode 100644 app/icons/edit-delete.svg create mode 100644 app/icons/edit-find.svg create mode 100644 app/icons/folder-open.svg create mode 100644 app/icons/format-list-unordered.svg create mode 100644 app/icons/go-down.svg create mode 100644 app/icons/help-hint.svg create mode 100644 app/icons/internet-services.svg create mode 100644 app/icons/list-add.svg create mode 100644 app/icons/mail-attachment.svg create mode 100644 app/icons/settings-configure.svg create mode 100644 app/icons/task-complete.svg create mode 100644 app/icons/text-x-generic.svg create mode 100644 app/qml/ImageStrip.qml create mode 100644 app/src/platform/android/AndroidTools.cpp create mode 100644 app/src/ui/SessionImages.cpp create mode 100644 app/src/ui/SessionImages.h create mode 100755 scripts/build_apk_android.sh create mode 100644 scripts/gradle-android-init.gradle diff --git a/.gitignore b/.gitignore index 12fb78b..1bb804d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,13 @@ vendor/sashidown/libsashidown-mingw.a .gen/ .build-tmp/ +# Android APK builds (scripts/build_apk_android.sh) and what gradle leaves +build-apk-android/ +*.apk +*.aab +app/android/.gradle/ +app/android/build/ +app/android/local.properties + # the engine as a static library (app/ links it; built by `make`) libcstcore.a diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 4f65395..b721666 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -76,13 +76,23 @@ if(CST_APP_GUI) # main.cpp loads) and let CMake generate the qmldir of qml/ for us. qt_policy(SET QTP0001 NEW) qt_policy(SET QTP0004 NEW) + # The front end's platform tools: one file per platform, the same split + # core/ToolProvider.cpp makes. Android brings its own (JNI, owner + # "app:android"); the other platforms still use the Linux file, whose + # registration entry point is simply not called there. + if(ANDROID) + set(CST_APP_PLATFORM_SRC src/platform/android/AndroidTools.cpp) + else() + set(CST_APP_PLATFORM_SRC src/platform/linux/NotifyTool.cpp) + endif() qt_add_executable(cst-agent-app src/main.cpp src/core/AgentCore.cpp src/core/ToolProvider.cpp src/ui/UiTest.cpp src/ui/ThemeIcons.cpp - src/platform/linux/NotifyTool.cpp) + src/ui/SessionImages.cpp + ${CST_APP_PLATFORM_SRC}) target_include_directories(cst-agent-app PRIVATE src "${CMAKE_CURRENT_SOURCE_DIR}/../include") target_compile_definitions(cst-agent-app PRIVATE CST_APP_VERSION="${PROJECT_VERSION}") @@ -96,6 +106,52 @@ if(CST_APP_GUI) set_target_properties(cst-agent-app PROPERTIES WIN32_EXECUTABLE TRUE) endif() + # --- Android (APK) ----------------------------------------------------- + # The engine is compiled in (top-level CMakeLists: -DCST_BUILD_TUI=OFF, + # -DCST_BUILD_WEBUI=OFF, so src/webui_stub.c stands in for the server and + # nothing listens on a port). The manifest, icons and strings live in + # app/android/, which androiddeployqt copies next to its own gradle + # project. minSdk 24 is the engine's own API_LEVEL (scripts/ + # build_third_party_android.sh builds curl/mbedTLS for android-24). + if(ANDROID) + file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/../include/cst_version.h" + _cst_ver_lines REGEX "#define CST_VERSION_(MAJOR|MINOR|PATCH) ") + foreach(_line ${_cst_ver_lines}) + if(_line MATCHES "CST_VERSION_MAJOR ([0-9]+)") + set(_cst_vmaj "${CMAKE_MATCH_1}") + elseif(_line MATCHES "CST_VERSION_MINOR ([0-9]+)") + set(_cst_vmin "${CMAKE_MATCH_1}") + elseif(_line MATCHES "CST_VERSION_PATCH ([0-9]+)") + set(_cst_vpat "${CMAKE_MATCH_1}") + endif() + endforeach() + set_target_properties(cst-agent-app PROPERTIES + QT_ANDROID_PACKAGE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/android" + QT_ANDROID_MIN_SDK_VERSION 24 + QT_ANDROID_TARGET_SDK_VERSION 35 + QT_ANDROID_VERSION_NAME "${_cst_vmaj}.${_cst_vmin}.${_cst_vpat}" + QT_ANDROID_VERSION_CODE 1 + QT_ANDROID_ABIS "${ANDROID_ABI}") + message(STATUS "app: Android package " + "${_cst_vmaj}.${_cst_vmin}.${_cst_vpat}, ABI ${ANDROID_ABI}, " + "minSdk 24, targetSdk 35") + + # Signing: androiddeployqt signs the APK itself and refuses a signed + # gradle output ("Could not copy unsigned APK"), so the keystore goes + # here, not into gradle. The path, alias and passwords arrive through + # the QT_ANDROID_KEYSTORE_* environment variables; + # scripts/build_apk_android.sh points them at the standard Android debug + # key it creates on first use, and CST_ANDROID_KEYSTORE overrides it. + if(DEFINED ENV{QT_ANDROID_KEYSTORE_PATH} AND NOT "$ENV{QT_ANDROID_KEYSTORE_PATH}" STREQUAL "") + set(QT_ANDROID_SIGN_APK TRUE) + message(STATUS "app: signing the APK with " + "$ENV{QT_ANDROID_KEYSTORE_PATH} ($ENV{QT_ANDROID_KEYSTORE_ALIAS})") + else() + message(STATUS "app: QT_ANDROID_KEYSTORE_PATH unset -- the APK stays " + "unsigned and 'adb install' will refuse it (INSTALL_PARSE_FAILED_NO_CERTIFICATES)") + endif() + endif() + # Theme is read by every component, so it is a QML singleton (one instance, # no object needed in the tree). set_source_files_properties(qml/Theme.qml PROPERTIES QT_QML_SINGLETON_TYPE TRUE) @@ -105,7 +161,7 @@ if(CST_APP_GUI) # requires absolute URLs" on every start). foreach(_qml Main ChatView Sidebar SessionRow MessageCard ToolCard ThinkBlock NoteLine TodoPanel Composer ConsentDialog AskDialog SettingsDialog - ThemeIcon Theme) + ThemeIcon Theme ImageStrip) set_source_files_properties(qml/${_qml}.qml PROPERTIES QT_RESOURCE_ALIAS ${_qml}.qml) endforeach() qt_add_qml_module(cst-agent-app @@ -126,6 +182,26 @@ if(CST_APP_GUI) qml/AskDialog.qml qml/SettingsDialog.qml qml/ThemeIcon.qml + qml/ImageStrip.qml qml/Theme.qml) + + # The icons the window asks for, compiled in (app/icons, Breeze's scalable + # names copied at 24@3x). The desktop finds them in the platform's icon + # theme and only falls back to these (ThemeIcons.cpp); a machine without + # any theme -- Android -- has nothing else, and before this the QML glyph + # fallback was all that showed, which the phone's fonts cannot render. + set(CST_APP_ICONS + go-down mail-attachment application-menu settings-configure edit-copy + edit-delete arrow-down arrow-right list-add task-complete box + document-edit text-x-generic folder-open edit-find code-context + internet-services format-list-unordered help-hint) + set(_cst_icon_files "") + foreach(_icon ${CST_APP_ICONS}) + list(APPEND _cst_icon_files "${CMAKE_CURRENT_SOURCE_DIR}/icons/${_icon}.svg") + endforeach() + qt_add_resources(cst-agent-app "cst_icons" + PREFIX "/cst-icons" + BASE "${CMAKE_CURRENT_SOURCE_DIR}/icons" + FILES ${_cst_icon_files}) endif() endif() diff --git a/app/README.md b/app/README.md index 7341467..bcbd694 100644 --- a/app/README.md +++ b/app/README.md @@ -212,6 +212,27 @@ is refused with a reason, the composer shows one chip, "clear all" empties the list, and — with an 8×8 image put on the clipboard — `clipboardHasImage()` and `attachClipboard()` behave. +## Images + +What a session keeps as a picture is shown, not only stored: a file +`view_image` returned, an image an MCP server sent, a screenshot the user +attached or pasted. Each one appears as a thumbnail in the transcript -- a +prompt's pictures under the message, a tool's under its card -- with the name, +size and origin in the tooltip. The height is capped so a 4K screenshot cannot +push the conversation off the screen, and a file the store no longer has says +which image it was instead of leaving a hole. + +The view never touches the filesystem on its own authority: the thumbnails +come from `image://cstattach/` (`app/src/ui/SessionImages.cpp`), +where the name is the one the engine wrote next to the session and is checked +with the engine's own `cst_attach_rel_ok()`, so only the session's attachment +directory can be read. The picture is decoded at the size of its thumbnail +rather than at its full resolution, and the provider's directory follows the +session: `AgentCore::attachmentDir()` is re-read on every switch. + +A document (PDF, UTF-8 text) still arrives as its name in the message text -- +those are sent to the model as text, not as a picture. + ## Transcript, tool calls, streaming text - A tool call is a card of its own, and the call **id** is what pairs it with @@ -307,7 +328,7 @@ GUI only through queued signals. |---|---|---| | Linux | `cmake -S app -B build-app` (libcurl via pkg-config) | clipboard (shell helpers), notification (`notify-send` / D-Bus) | | Windows | Qt's MinGW kit, `cmake -G Ninja -DCMAKE_PREFIX_PATH=/lib/cmake`; the engine needs libcurl for Windows (see `docs/README-windows.md`) | notification via the WinRT/tray API, clipboard through the core's Win32 path | -| Android | engine compiled with the NDK inside the APK (`-DCST_TUI=0`, `CST_NO_WEBUI=1`); the app talks to Android APIs through Qt (JNI) | notification (Android), share sheet, clipboard, file picker | +| Android | engine compiled with the NDK inside the APK (`scripts/build_apk_android.sh`, `-DCST_TUI=0`, `CST_NO_WEBUI=1`); no port, QML talks to `AgentCore` in-process | none yet: `app/src/platform/android/AndroidTools.cpp` is a stub, so notification, share sheet, clipboard and the file picker are still missing | | macOS | Qt from the installer, libcurl from the system | notification (`osascript`), clipboard (`pbpaste`/`pbcopy`) | Android's long-term shape: the core lives in the APK (no Termux needed), the @@ -328,8 +349,12 @@ The build flag that keeps Unix helpers out of the picture is - [x] attachments: picker, drop on the window, Ctrl+V image paste - [x] icons from the desktop's icon theme (Breeze), with a glyph fallback - [x] contrast pass: faint text and selected rows are readable +- [x] images in the transcript: tool results and attachments are drawn as + thumbnails (`image://cstattach/`, see the Images section) +- [x] Android APK: the engine is linked in (no port), builds, installs under + `xyz.whcself.cstagent` and runs on the device (see `docs/android.md`) - [ ] a real turn driven through the window (needs a provider key) -- [ ] tool results that carry images (they are stored, not drawn yet) +- [ ] Android platform tools (JNI: notification, share sheet, clipboard, file + picker), the back gesture, the icon glyphs on the phone - [ ] Kirigami (the rest of the Breeze look: colors, controls), tray, global shortcuts -- [ ] Android APK (engine in-process, JNI platform tools) diff --git a/app/android/AndroidManifest.xml b/app/android/AndroidManifest.xml new file mode 100644 index 0000000..e812bb1 --- /dev/null +++ b/app/android/AndroidManifest.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/android/res/drawable/ic_launcher.xml b/app/android/res/drawable/ic_launcher.xml new file mode 100644 index 0000000..6399adb --- /dev/null +++ b/app/android/res/drawable/ic_launcher.xml @@ -0,0 +1,30 @@ + + + + + + + + + + diff --git a/app/android/res/values/strings.xml b/app/android/res/values/strings.xml new file mode 100644 index 0000000..ebb2a01 --- /dev/null +++ b/app/android/res/values/strings.xml @@ -0,0 +1,5 @@ + + + + CST-agent + diff --git a/app/icons/application-menu.svg b/app/icons/application-menu.svg new file mode 100644 index 0000000..46546db --- /dev/null +++ b/app/icons/application-menu.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/arrow-down.svg b/app/icons/arrow-down.svg new file mode 100644 index 0000000..e3e315c --- /dev/null +++ b/app/icons/arrow-down.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/arrow-right.svg b/app/icons/arrow-right.svg new file mode 100644 index 0000000..16e6318 --- /dev/null +++ b/app/icons/arrow-right.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/box.svg b/app/icons/box.svg new file mode 100644 index 0000000..d907430 --- /dev/null +++ b/app/icons/box.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/app/icons/code-context.svg b/app/icons/code-context.svg new file mode 100644 index 0000000..809a23e --- /dev/null +++ b/app/icons/code-context.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/document-edit.svg b/app/icons/document-edit.svg new file mode 100644 index 0000000..7f6273d --- /dev/null +++ b/app/icons/document-edit.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/edit-copy.svg b/app/icons/edit-copy.svg new file mode 100644 index 0000000..4a6a05c --- /dev/null +++ b/app/icons/edit-copy.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/app/icons/edit-delete.svg b/app/icons/edit-delete.svg new file mode 100644 index 0000000..6f10a57 --- /dev/null +++ b/app/icons/edit-delete.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/edit-find.svg b/app/icons/edit-find.svg new file mode 100644 index 0000000..11428d1 --- /dev/null +++ b/app/icons/edit-find.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/folder-open.svg b/app/icons/folder-open.svg new file mode 100644 index 0000000..fe9631b --- /dev/null +++ b/app/icons/folder-open.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/format-list-unordered.svg b/app/icons/format-list-unordered.svg new file mode 100644 index 0000000..d1b401c --- /dev/null +++ b/app/icons/format-list-unordered.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/go-down.svg b/app/icons/go-down.svg new file mode 100644 index 0000000..e3e315c --- /dev/null +++ b/app/icons/go-down.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/help-hint.svg b/app/icons/help-hint.svg new file mode 100644 index 0000000..63ad186 --- /dev/null +++ b/app/icons/help-hint.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/internet-services.svg b/app/icons/internet-services.svg new file mode 100644 index 0000000..c3390f1 --- /dev/null +++ b/app/icons/internet-services.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/app/icons/list-add.svg b/app/icons/list-add.svg new file mode 100644 index 0000000..36a7a1f --- /dev/null +++ b/app/icons/list-add.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/mail-attachment.svg b/app/icons/mail-attachment.svg new file mode 100644 index 0000000..08bdfa1 --- /dev/null +++ b/app/icons/mail-attachment.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/settings-configure.svg b/app/icons/settings-configure.svg new file mode 100644 index 0000000..24aebea --- /dev/null +++ b/app/icons/settings-configure.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/task-complete.svg b/app/icons/task-complete.svg new file mode 100644 index 0000000..1a0617a --- /dev/null +++ b/app/icons/task-complete.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/icons/text-x-generic.svg b/app/icons/text-x-generic.svg new file mode 100644 index 0000000..a5d3d1b --- /dev/null +++ b/app/icons/text-x-generic.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/app/qml/ChatView.qml b/app/qml/ChatView.qml index 988a849..cdb37b3 100644 --- a/app/qml/ChatView.qml +++ b/app/qml/ChatView.qml @@ -40,6 +40,7 @@ Item { role: model.role text: model.text complete: model.complete + images: model.images !== undefined ? model.images : "" /* html comes from the streaming renderer (Main.qml) or, * for a replayed session, from here once. */ html: model.html.length > 0 ? model.html @@ -65,6 +66,7 @@ Item { output: model.output exitCode: model.exitCode hasResult: model.complete + images: model.images !== undefined ? model.images : "" } } Component { diff --git a/app/qml/ImageStrip.qml b/app/qml/ImageStrip.qml new file mode 100644 index 0000000..b885fb8 --- /dev/null +++ b/app/qml/ImageStrip.qml @@ -0,0 +1,110 @@ +import QtQuick +import QtQuick.Controls +import QtQuick.Layouts + +// The images a message or a tool call carries: thumbnails from the session's +// own store (image://cstattach/, app/src/ui/SessionImages.cpp). The names are +// the ones the engine wrote next to the session -- view_image, an MCP image, a +// picture the user attached -- never a path the view invented. +// +// One row of thumbnails: the height is capped so a screenshot cannot push the +// conversation off the screen, the width follows the picture's shape. +Flow { + id: strip + objectName: "imageStrip" + /* One line per picture, written by the engine side (AgentCore.cpp: + * "|||x"). A row carries this as text + * because a ListModel role cannot hold a list. */ + property string text: "" + property var images: decode(text) + spacing: 6 + + function decode(t) { + const out = [] + const lines = String(t).split("\n") + for (let i = 0; i < lines.length; i++) { + if (!lines[i].length) + continue + const f = lines[i].split("|") + if (!f[0].length) + continue + let w = 0, h = 0 + const dim = String(f[3] || "").split("x") + if (dim.length === 2) { + w = parseInt(dim[0], 10) || 0 + h = parseInt(dim[1], 10) || 0 + } + out.push({ url: "image://cstattach/" + f[0], rel: f[0], + name: f[1] || f[0], origin: f[2] || "", w: w, h: h }) + } + return out + } + + Repeater { + model: strip.images + + delegate: Rectangle { + id: thumb + required property var modelData + required property int index + readonly property real ratio: { + const w = Number(modelData.w), h = Number(modelData.h) + return (w > 0 && h > 0) ? w / h : 1.5 + } + readonly property int cap: 150 // the tallest a thumbnail gets + objectName: "imageThumb" + index + width: Math.max(56, Math.min(Math.round(cap * ratio), 320)) + height: Math.max(40, Math.min(cap, Math.round(width / ratio))) + radius: Theme.radiusSm + color: Theme.bg + border.width: 1 + border.color: mouse.containsMouse ? Theme.accentDim : Theme.border + clip: true + + Image { + id: pic + anchors.fill: parent + anchors.margins: 2 + source: String(thumb.modelData.url) + sourceSize.width: 512 + sourceSize.height: 512 + fillMode: Image.PreserveAspectFit + /* Not asynchronous: the provider already hands back a + * thumbnail-sized picture, so there is nothing to wait for. */ + asynchronous: false + visible: status === Image.Ready + } + + // A file the store no longer has is not worth shouting about: the + // frame says which image it was. + Text { + anchors.centerIn: parent + width: parent.width - 8 + visible: pic.status === Image.Error + text: String(thumb.modelData.name) + elide: Text.ElideMiddle + horizontalAlignment: Text.AlignHCenter + color: Theme.textFaint + font.pixelSize: Theme.smallSize + } + + MouseArea { + id: mouse + anchors.fill: parent + hoverEnabled: true + acceptedButtons: Qt.NoButton + } + ToolTip.visible: mouse.containsMouse + ToolTip.delay: 400 + ToolTip.text: { + const m = thumb.modelData + let s = String(m.name) + if (m.w > 0 && m.h > 0) + s += " · " + m.w + "×" + m.h + if (m.origin && m.origin !== "user") + s += " · " + m.origin + return s + } + } + } +} diff --git a/app/qml/Main.qml b/app/qml/Main.qml index 1f87de9..ab26331 100644 --- a/app/qml/Main.qml +++ b/app/qml/Main.qml @@ -61,7 +61,7 @@ ApplicationWindow { function addRow(kind, role, state) { transcript.append({ kind: kind, role: role, text: "", html: "", complete: false, name: "", callId: "", state: state, detail: "", output: "", - exitCode: 0 }) + exitCode: 0, images: "" }) } // The engine streams one kind at a time: close whatever was open before. @@ -112,7 +112,7 @@ ApplicationWindow { function submit(text) { transcript.append({ kind: "message", role: "user", text: text, html: "", complete: true, name: "", callId: "", state: "", detail: "", output: "", - exitCode: 0 }) + exitCode: 0, images: "" }) agent.send(text) } @@ -545,6 +545,29 @@ ApplicationWindow { win.statusText = "" } + /* The images the prompt just carried (AgentCore::promptImages): the + * composer has appended the user row, so they attach to it. */ + function enginePromptImages(images) { + if (!images.length) + return + for (let i = transcript.count - 1; i >= 0; i--) { + const r = transcript.get(i) + if (r.kind === "message" && r.role === "user") { + transcript.setProperty(i, "images", images) + return + } + } + } + + /* Images a tool produced, on the card of the call that made them. */ + function engineToolImages(callId, images) { + if (!images.length) + return + const i = win.findTool(callId, "") + if (i >= 0) + transcript.setProperty(i, "images", images) + } + /* For --uitest: one transcript row, or null. */ function rowAt(i) { return (i >= 0 && i < transcript.count) ? transcript.get(i) : null @@ -584,6 +607,14 @@ ApplicationWindow { win.engineToolResult(id, name, exitCode, output) } + function onToolImages(callId, images) { + win.engineToolImages(callId, images) + } + + function onPromptImages(images) { + win.enginePromptImages(images) + } + function onPermissionRequested(tool, detail, preview) { consent.tool = tool consent.detail = detail diff --git a/app/qml/MessageCard.qml b/app/qml/MessageCard.qml index 11d84fb..424ad73 100644 --- a/app/qml/MessageCard.qml +++ b/app/qml/MessageCard.qml @@ -11,6 +11,7 @@ Rectangle { property string text: "" property string html: "" property bool complete: true + property string images: "" radius: Theme.radius color: role === "user" ? Theme.surfaceAlt : Theme.surface @@ -105,5 +106,12 @@ Rectangle { wrapMode: Text.Wrap onLinkActivated: function (link) { Qt.openUrlExternally(link) } } + + // What the prompt carried: pictures from the store, not their names. + ImageStrip { + Layout.fillWidth: true + visible: card.images.length > 0 + text: card.images + } } } diff --git a/app/qml/ToolCard.qml b/app/qml/ToolCard.qml index f944b6f..ab5ec53 100644 --- a/app/qml/ToolCard.qml +++ b/app/qml/ToolCard.qml @@ -14,6 +14,7 @@ Rectangle { property int exitCode: 0 property bool hasResult: false property bool expanded: false + property string images: "" readonly property bool running: state === "pending" || state === "ready" || state === "running" readonly property color stateColor: state === "denied" ? Theme.err @@ -125,6 +126,14 @@ Rectangle { } } + // What the tool made (view_image, an MCP image): shown as it is, with + // the output left folded -- the picture is the result. + ImageStrip { + Layout.fillWidth: true + visible: card.images.length > 0 + text: card.images + } + // Folded view: the first line of the result, so a done card still says // something without being opened. Text { diff --git a/app/src/core/AgentCore.cpp b/app/src/core/AgentCore.cpp index 49aaaee..118b5f8 100644 --- a/app/src/core/AgentCore.cpp +++ b/app/src/core/AgentCore.cpp @@ -17,6 +17,7 @@ #include #include "cst_agent.h" +#include "cst_attach.h" #include "cst_models.h" #include "cst_provider_switch.h" #include "cst_web_md.h" @@ -53,6 +54,7 @@ AgentCore::AgentCore(QObject *parent) : QObject(parent) { m_hooks.on_think_delta = hookThink; m_hooks.on_tool_event = hookToolEvent; m_hooks.on_tool_result = hookToolResult; + m_hooks.on_tool_images = hookToolImages; m_hooks.on_todo = hookTodo; m_hooks.on_permission = hookPermission; m_hooks.on_ask_user = hookAskUser; @@ -172,7 +174,12 @@ void AgentCore::close() { void AgentCore::send(const QString &prompt) { if (!m_agent || m_running || prompt.trimmed().isEmpty()) return; + /* The prompt takes the pending images with it: the view shows them on the + * message the composer just appended, before the turn can say anything. */ + const QString sending = pendingImages(); setRunning(true); + if (!sending.isEmpty()) + emit promptImages(sending); m_turn = new Turn(this, prompt); m_turn->start(); } @@ -314,9 +321,41 @@ QVariantMap sessionRow(const QString &kind, const QString &role) { r["detail"] = QString(); r["output"] = QString(); r["exitCode"] = 0; + r["images"] = QString(); return r; } +/* One field of one picture line: names come from the user and from tool + * output, so a separator may not split a line in two. */ +QString field(const char *s) { + QString out = utf8(s); + out.replace(QLatin1Char('|'), QLatin1Char('/')); + out.replace(QLatin1Char('\n'), QLatin1Char(' ')); + return out; +} + +/* The images of one message or tool result, one line per picture: + * + * |||x + * + * A list role is not an option: a ListModel keeps what fits in a QVariant of + * a simple type -- a list reads back as undefined there, and setProperty drops + * the role -- so the rows carry this text and ImageStrip.qml decodes it. A + * document (PDF, text) is not a picture and stays out of it. */ +QString imagesText(const cst_attachment_t *list, int n) { + QStringList lines; + for (int i = 0; list && i < n; i++) { + const cst_attachment_t *a = &list[i]; + if (!cst_attach_is_image(a)) + continue; + lines << QStringLiteral("%1|%2|%3|%4x%5") + .arg(field(a->rel), field(a->name), field(a->origin)) + .arg(a->width) + .arg(a->height); + } + return lines.join(QLatin1Char('\n')); +} + /* The open tool row this result belongs to, -1 when none. Rows are paired by * call id first: the same tool may run twice in a turn, and pairing by name * would hang the first card on "running" forever. Only a call without an id @@ -354,6 +393,20 @@ QString AgentCore::sessionId() const { return utf8(id); } +/* Where the session keeps its images: what image://cstattach/ serves, and + * what the view asks for when the session changes. */ +QString AgentCore::attachmentDir() const { + if (!m_agent) + return QString(); + const char *sess = cst_agent_session_path(m_agent); + if (!sess || !sess[0]) + return QString(); + char dir[4096] = ""; + if (cst_attach_dir(sess, dir, sizeof(dir)) != 0) + return QString(); + return utf8(dir); +} + QVariantList AgentCore::sessions() const { QVariantList out; if (m_workspace.isEmpty()) @@ -404,12 +457,14 @@ QVariantList AgentCore::history() const { r["name"] = name; r["callId"] = callId; r["output"] = utf8(m->content); + r["images"] = imagesText(m->attach, m->nattach); r["state"] = "done"; r["complete"] = true; rows.append(r); } else { QVariantMap r = rows.at(at).toMap(); r["output"] = utf8(m->content); + r["images"] = imagesText(m->attach, m->nattach); r["state"] = "done"; r["complete"] = true; rows[at] = r; @@ -426,13 +481,16 @@ QVariantList AgentCore::history() const { if (m->role == CST_ROLE_USER) { QVariantMap r = sessionRow(QStringLiteral("message"), QStringLiteral("user")); QString text = utf8(m->content); - if (m->nattach > 0) { - QStringList names; - for (int a = 0; a < m->nattach; a++) + /* A document keeps its name in the text; a picture is shown instead + * (the images below), so only the rest is worth naming. */ + QStringList names; + for (int a = 0; a < m->nattach; a++) + if (!cst_attach_is_image(&m->attach[a])) names << utf8(m->attach[a].name); + if (!names.isEmpty()) text += QStringLiteral("\n\n[attached: %1]").arg(names.join(", ")); - } r["text"] = text; + r["images"] = imagesText(m->attach, m->nattach); r["complete"] = true; rows.append(r); continue; @@ -802,6 +860,30 @@ QVariantMap AgentCore::usage() const { /* --- attachments ---------------------------------------------------------- */ +/* What the prompt about to be sent carries, ready for the view. The stored + * name is the engine's (the file already lives next to the session), the + * display name is the one the window kept when it copied the file in. */ +QString AgentCore::pendingImages() const { + QStringList lines; + if (!m_agent) + return QString(); + const int n = cst_agent_pending_count(m_agent); + for (int i = 0; i < n; i++) { + if (!cst_agent_pending_is_image(m_agent, i)) + continue; + const char *rel = cst_agent_pending_rel(m_agent, i); + if (!rel || !rel[0]) + continue; + const QString name = + i < m_attachNames.size() ? m_attachNames.at(i) : utf8(rel); + /* The size is not known before the engine stores the file; the strip + * falls back to a picture's usual shape until it is. */ + lines << QStringLiteral("%1|%2|user|0x0") + .arg(field(rel), field(name.toUtf8().constData())); + } + return lines.join(QLatin1Char('\n')); +} + QStringList AgentCore::pendingAttachments() const { QStringList out; if (!m_agent) @@ -937,6 +1019,16 @@ void AgentCore::hookToolResult(const ctl_tool_call_t *call, const char *out, [self, id, n, exit_code, o] { emit self->toolResult(id, n, exit_code, o); }); } +void AgentCore::hookToolImages(const ctl_tool_call_t *call, + const cst_attachment_t *list, int n, void *ud) { + AgentCore *self = static_cast(ud); + const QString images = imagesText(list, n); + if (images.isEmpty()) + return; /* a call that made no picture needs no signal */ + const QString id = QString::fromUtf8(call && call->id ? call->id : ""); + self->post([self, id, images] { emit self->toolImages(id, images); }); +} + void AgentCore::hookTodo(const char *text, void *ud) { AgentCore *self = static_cast(ud); const QString t = QString::fromUtf8(text ? text : ""); diff --git a/app/src/core/AgentCore.h b/app/src/core/AgentCore.h index b5f7e21..67cf3c9 100644 --- a/app/src/core/AgentCore.h +++ b/app/src/core/AgentCore.h @@ -63,7 +63,8 @@ class AgentCore : public QObject { Q_INVOKABLE QVariantList sessions() const; /* The open session as transcript rows, in exactly the shape the live * signals build (kind, role, text, name, state, detail, output, exitCode, - * complete), so a reopened session renders like the turn that wrote it. */ + * complete, images), so a reopened session renders like the turn that wrote + * it. */ Q_INVOKABLE QVariantList history() const; /* None of these run during a turn; all return false when one does. */ Q_INVOKABLE bool newSession(); @@ -128,6 +129,16 @@ class AgentCore : public QObject { * clipboard) or stay with the editor (text). */ Q_INVOKABLE bool clipboardHasImage() const; Q_INVOKABLE void clearPending(); + /* The images among the pending attachments, in the engine's order, in the + * same text history() and toolImages() carry (see imagesText()): one line + * per picture, "|||x". Empty for a PDF or + * a text file, and for a picture whose stored name the engine did not + * report. */ + Q_INVOKABLE QString pendingImages() const; + /* Where the current session keeps its stored images: the directory + * image://cstattach/ serves (SessionImages.cpp). "" until the engine is + * open. It follows the session. */ + Q_INVOKABLE QString attachmentDir() const; /* 1 the model is shown images, 0 it is not (they go as notes), -1 unknown. */ Q_INVOKABLE int seesImages() const; @@ -176,6 +187,12 @@ class AgentCore : public QObject { const QString &detail); void toolResult(const QString &id, const QString &name, int exitCode, const QString &output); + /* Images a tool produced (view_image, an MCP image), right after its + * result, as the text ImageStrip.qml decodes. */ + void toolImages(const QString &callId, const QString &images); + /* The images the prompt that was just sent carries, in the same shape, + * fired before the turn's first delta. */ + void promptImages(const QString &images); void todoChanged(const QString &text); void permissionRequested(const QString &tool, const QString &detail, const QString &preview); @@ -203,6 +220,8 @@ class AgentCore : public QObject { const char *detail, void *ud); static void hookToolResult(const ctl_tool_call_t *call, const char *out, size_t len, int exit_code, void *ud); + static void hookToolImages(const ctl_tool_call_t *call, + const cst_attachment_t *list, int n, void *ud); static void hookTodo(const char *text, void *ud); static int hookPermission(const char *tool, const char *detail, const char *preview, void *ud); diff --git a/app/src/main.cpp b/app/src/main.cpp index d52ced3..03be29e 100644 --- a/app/src/main.cpp +++ b/app/src/main.cpp @@ -27,9 +27,24 @@ #include "core/AgentCore.h" #include "core/ToolProvider.h" #include "ui/UiTest.h" +#include "ui/SessionImages.h" #include "ui/ThemeIcons.h" #include "cst_version.h" +#if defined(Q_OS_ANDROID) +/* Where an Android app is allowed to write: its own data directory. There is + * no usable $HOME, no writable "/" (QDir::currentPath()) and no /tmp, while + * the engine wants a config dir ($XDG_CONFIG_HOME/cst-agent, os_config_dir()) + * and a scratch dir (the first writable entry of os_temp_dir()'s list). All + * of them are pointed here -- once, before the engine exists. */ +QString androidRoot() { + const QString root = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return root.isEmpty() ? QDir::tempPath() + QStringLiteral("/cst-agent") + : root; +} +#endif + namespace { int runSelftest() { @@ -141,8 +156,13 @@ int main(int argc, char **argv) { parser.process(app); QString ws = parser.value(wsOpt); - if (ws.isEmpty()) + if (ws.isEmpty()) { +#if defined(Q_OS_ANDROID) + ws = androidRoot() + QStringLiteral("/workspace"); +#else ws = QDir::currentPath(); +#endif + } ws = QFileInfo(ws).absoluteFilePath(); /* --uitest drives a scratch workspace when none was given: the session @@ -160,6 +180,17 @@ int main(int argc, char **argv) { } } +#if defined(Q_OS_ANDROID) + /* Before the engine is created: its settings, credentials and session store + * all resolve their paths from the environment. */ + const QString root = androidRoot(); + for (const QString &sub : {QStringLiteral("config"), QStringLiteral("tmp")}) + QDir().mkpath(root + QLatin1Char('/') + sub); + qputenv("XDG_CONFIG_HOME", (root + QStringLiteral("/config")).toUtf8()); + qputenv("TMPDIR", (root + QStringLiteral("/tmp")).toUtf8()); + qputenv("HOME", root.toUtf8()); +#endif + AgentCore core; QString err; if (!core.open(ws, parser.value(providerOpt), parser.value(modelOpt), @@ -195,6 +226,13 @@ int main(int argc, char **argv) { QQmlApplicationEngine engine; /* image://csttheme/ plus the desktop's icon theme behind it. */ app::installThemeIcons(&engine); + /* image://cstattach/: the session's own store, so the pictures a tool made + * (view_image, an MCP image) and the ones the user attached are shown, not + * only named. The directory follows the session. */ + app::SessionImages *images = app::installSessionImages(&engine); + images->setDir(core.attachmentDir()); + QObject::connect(&core, &AgentCore::sessionChanged, images, + [images, &core] { images->setDir(core.attachmentDir()); }); engine.rootContext()->setContextProperty(QStringLiteral("agent"), &core); engine.rootContext()->setContextProperty(QStringLiteral("initialWidth"), winW); engine.rootContext()->setContextProperty(QStringLiteral("initialHeight"), winH); diff --git a/app/src/platform/android/AndroidTools.cpp b/app/src/platform/android/AndroidTools.cpp new file mode 100644 index 0000000..0eeb838 --- /dev/null +++ b/app/src/platform/android/AndroidTools.cpp @@ -0,0 +1,43 @@ +/* Platform tools on Android, implemented by the app: the same contract as + * app/src/platform/linux/NotifyTool.cpp, with Android APIs behind it. + * + * The engine only ever sees the capability: `platforms = kAndroidOnly` (the + * engine reports "android" when it was compiled with the NDK, see + * src/tools/registry.c: cst_platform_name()) plus an `available` probe decide + * whether a tool reaches the prompt and tools[] at all -- a Termux build on + * the same device reports "termux" and gets the android tools too, an APK + * never gets the desktop ones. `owner = "app:android"` is what + * unregisterPlatformTools() drops again. + * + * WHERE THE CODE RUNS: `run_ex` is called on the turn's worker thread, never + * on the GUI thread. JNI calls are fine from any thread (QJniEnvironment + * attaches it), Qt GUI objects are not -- a tool that needs the widget tree + * has to hop back with a blocking queued call. + * + * WHAT GOES HERE (one at a time, each with its own probe): + * notification_send NotificationChannel + NotificationManager through + * JNI; hidden until POST_NOTIFICATIONS is granted + * (Android 13+), which the UI asks for on first use. + * clipboard_get/set the phone clipboard through Qt's QClipboard, taking + * the name of the desktop tool that cannot work here + * (see app/README.md: a front end may shadow a built-in + * that has no working backend on its host). + * share ACTION_SEND of a file or text through the system share + * sheet -- the phone's answer to "show this to the user". + * + * Nothing is registered yet: the first APK ships the engine, the window and + * the tool contract, and the tools above arrive next (docs/android.md). */ +#include + +#include "core/AgentCore.h" +#include "core/ToolProvider.h" + +namespace app { + +int platformRegisterAndroid(AgentCore *core) { + if (!core) + return 0; + return 0; /* no Android tools yet */ +} + +} // namespace app diff --git a/app/src/ui/SessionImages.cpp b/app/src/ui/SessionImages.cpp new file mode 100644 index 0000000..a48414f --- /dev/null +++ b/app/src/ui/SessionImages.cpp @@ -0,0 +1,81 @@ +#include "ui/SessionImages.h" + +#include +#include +#include + +#include "cst_attach.h" + +namespace app { +namespace { + +/* "img-1a2b3c4d.png?w=160": the stored name, then what the view asked for. + * The size comes from the QML side as sourceSize, which Qt passes in as + * requestedSize; the query string is accepted too, so a plain URL can ask. */ +QString relOf(const QString &id) { + const int q = id.indexOf(QLatin1Char('?')); + return q < 0 ? id : id.left(q); +} + +} // namespace + +SessionImages::SessionImages() + : QQuickImageProvider(QQuickImageProvider::Image) {} + +void SessionImages::setDir(const QString &dir) { + QMutexLocker lock(&m_mu); + m_dir = dir; +} + +QImage SessionImages::requestImage(const QString &id, QSize *size, + const QSize &requestedSize) { + const QString rel = relOf(id); + if (!cst_attach_rel_ok(rel.toUtf8().constData())) + return QImage(); /* QML sees Image.Error and falls back to the name */ + QString dir; + { + QMutexLocker lock(&m_mu); + dir = m_dir; + } + if (dir.isEmpty()) + return QImage(); + const QString path = dir + QLatin1Char('/') + rel; + if (!QFileInfo::exists(path)) + return QImage(); + + QImageReader reader(path); + reader.setAutoTransform(true); + const QSize full = reader.size(); + if (!full.isValid()) { + /* No size in the header (or not a picture at all): hand back what Qt + * makes of it and let the view scale it. */ + const QImage img = reader.read(); + if (size) + *size = img.size(); + return img; + } + /* Shrink to what the view shows, never up: a screenshot is decoded at the + * size of its thumbnail instead of at 4K. */ + const bool smaller = requestedSize.isValid() && requestedSize.width() > 0 && + requestedSize.height() > 0 && + (full.width() > requestedSize.width() || + full.height() > requestedSize.height()); + if (smaller) { + const QSize scaled = full.scaled(requestedSize, Qt::KeepAspectRatio); + reader.setScaledSize(scaled); + if (size) + *size = scaled; + } else if (size) { + *size = full; + } + return reader.read(); +} + +SessionImages *installSessionImages(QQmlApplicationEngine *engine) { + SessionImages *images = new SessionImages; + if (engine) + engine->addImageProvider(QStringLiteral("cstattach"), images); + return images; +} + +} // namespace app diff --git a/app/src/ui/SessionImages.h b/app/src/ui/SessionImages.h new file mode 100644 index 0000000..bee152c --- /dev/null +++ b/app/src/ui/SessionImages.h @@ -0,0 +1,45 @@ +/* The session's stored images as a QML image provider: + * + * image://cstattach/ (see cst_attach.h: "img-1a2b3c4d.png") + * + * The files live in the current session's attachment directory, which is what + * AgentCore::attachmentDir() names. Only that directory is served, and only + * names that pass the engine's own cst_attach_rel_ok() -- a transcript can + * carry hand-edited JSON, and nothing in it may reach outside the store. + * + * requestImage() runs on Quick's render thread while setDir() comes from the + * window's thread, hence the mutex. */ +#ifndef CST_APP_SESSION_IMAGES_H +#define CST_APP_SESSION_IMAGES_H + +#include +#include +#include +#include + +class QQmlApplicationEngine; + +namespace app { + +class SessionImages : public QQuickImageProvider { + public: + SessionImages(); + + /* The directory holding the current session's images; "" disables serving. */ + void setDir(const QString &dir); + + QImage requestImage(const QString &id, QSize *size, + const QSize &requestedSize) override; + + private: + mutable QMutex m_mu; + QString m_dir; +}; + +/* Registers image://cstattach/ on the engine and hands the provider back, so + * the caller can keep its directory in step with the agent's session. */ +SessionImages *installSessionImages(QQmlApplicationEngine *engine); + +} // namespace app + +#endif // CST_APP_SESSION_IMAGES_H diff --git a/app/src/ui/ThemeIcons.cpp b/app/src/ui/ThemeIcons.cpp index 77be9ba..a7deeb2 100644 --- a/app/src/ui/ThemeIcons.cpp +++ b/app/src/ui/ThemeIcons.cpp @@ -48,6 +48,21 @@ void adoptIconTheme() { } } +/* The bundled set: app/icons/*.svg, compiled in as :/cst-icons/.svg by + * app/CMakeLists.txt. A machine without an icon theme -- Android, where the + * Qt theme lookup finds nothing and QML was left showing its glyph fallback, + * which the phone fonts cannot draw -- gets the same icons from here. */ +QIcon bundledIcon(const QString &name) { + return QIcon(QStringLiteral(":/cst-icons/") + name + QStringLiteral(".svg")); +} + +QIcon iconForName(const QString &name) { + const QIcon themed = QIcon::fromTheme(name); + if (!themed.isNull()) + return themed; + return bundledIcon(name); +} + /* "?color=%23eff0f1": the tint, and nothing else so far. */ QColor colorOption(const QString &id, QString *name) { QColor color; @@ -85,7 +100,7 @@ QImage ThemeIconProvider::requestImage(const QString &id, QSize *size, if (name.isEmpty()) return QImage(); - const QIcon icon = QIcon::fromTheme(name); + const QIcon icon = iconForName(name); if (icon.isNull()) return QImage(); /* QML shows its fallback glyph */ @@ -116,8 +131,6 @@ void installThemeIcons(QQmlApplicationEngine *engine) { QString iconThemeName() { return QIcon::themeName(); } -bool themeIconAvailable(const QString &name) { - return QIcon::hasThemeIcon(name); -} +bool themeIconAvailable(const QString &name) { return !iconForName(name).isNull(); } } // namespace app diff --git a/app/src/ui/UiTest.cpp b/app/src/ui/UiTest.cpp index 240f43f..c855fe5 100644 --- a/app/src/ui/UiTest.cpp +++ b/app/src/ui/UiTest.cpp @@ -10,10 +10,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -27,6 +29,8 @@ #include #include "cst_agent.h" +#include "cst_attach.h" +#include "ui/SessionImages.h" #include "ui/ThemeIcons.h" namespace app { @@ -118,7 +122,26 @@ QString writeFixtureSession(const QString &ws, const QString &id) { memset(&m, 0, sizeof(m)); m.role = CST_ROLE_USER; m.content = strdup("what is here?"); + /* The prompt carries a picture, stored exactly the way the engine stores + * one (the file next to the session, the message naming it). */ + { + QImage pic(320, 200, QImage::Format_RGB32); + pic.fill(Qt::darkCyan); + QPainter painter(&pic); + painter.fillRect(QRect(0, 0, 160, 100), Qt::darkYellow); + painter.end(); + QByteArray png; + QBuffer buf(&png); + buf.open(QIODevice::WriteOnly); + pic.save(&buf, "PNG"); + char aerr[256] = ""; + m.attach = cst_attach_import_bytes(path, "shot.png", png.constData(), + static_cast(png.size()), aerr, + sizeof(aerr)); + m.nattach = m.attach ? 1 : 0; + } cst_session_append(s, &m); + cst_attachments_free(m.attach, m.nattach); memset(&m, 0, sizeof(m)); m.role = CST_ROLE_ASSISTANT; m.content = strdup("let me look. "); @@ -175,6 +198,12 @@ int runUiTest(AgentCore &core, const QString &size) { QQmlApplicationEngine engine; installThemeIcons(&engine); /* the window's icons, same as main() */ + /* image://cstattach/: the session's stored images, pointed at the session + * the engine has open (and re-pointed on every switch). */ + SessionImages *images = installSessionImages(&engine); + images->setDir(core.attachmentDir()); + QObject::connect(&core, &AgentCore::sessionChanged, images, + [images, &core] { images->setDir(core.attachmentDir()); }); engine.rootContext()->setContextProperty(QStringLiteral("agent"), &core); engine.rootContext()->setContextProperty(QStringLiteral("initialWidth"), width); engine.rootContext()->setContextProperty(QStringLiteral("initialHeight"), height); @@ -303,6 +332,36 @@ int runUiTest(AgentCore &core, const QString &size) { clickMouse(win, QStringLiteral("sessionRow:") + fixture); check(core.sessionId() == fixture && list && list->property("count").toInt() == 5, QStringLiteral("switching back replays it again")); + { + bool promptImage = false; + const QVariantList rows = core.history(); + for (const QVariant &v : rows) { + const QVariantMap r = v.toMap(); + if (r.value(QStringLiteral("role")).toString() == QLatin1String("user") && + r.value(QStringLiteral("images")).toString().contains( + QLatin1String("|"))) + promptImage = true; + } + check(promptImage, + QStringLiteral("a stored prompt image comes back with its row")); + /* The rows are in the model by now; the cards -- and the thumbnails in + * them -- are built on the view's next layout pass. A frame forces it, + * and that frame is also the picture a person looks at. */ + maybeShot(win, QStringLiteral("-images")); + QTest::qWait(150); + if (qEnvironmentVariableIsSet("CST_APP_DUMP")) { + QVariant v; + QMetaObject::invokeMethod(win, "rowJson", Q_RETURN_ARG(QVariant, v), + Q_ARG(QVariant, 0)); + std::printf("note: replayed row 0: %s\n", qPrintable(v.toString())); + std::printf("note: %d strips and %d thumbs in the tree\n", + countNamed(win->contentItem(), QStringLiteral("imageStrip")), + countNamed(win->contentItem(), QStringLiteral("imageThumb"))); + } + check(countNamed(win->contentItem(), QStringLiteral("imageThumb")) >= 1, + QStringLiteral("the replayed transcript shows it as a thumbnail (%1)") + .arg(countNamed(win->contentItem(), QStringLiteral("imageThumb")))); + } /* --- provider, model, key: the settings dialog ------------------------ */ check(core.providers().size() > 0 && @@ -558,6 +617,72 @@ int runUiTest(AgentCore &core, const QString &size) { check(false, QStringLiteral("assistant deltas are reachable")); } + /* --- the image provider: only the session's own files ----------------- */ + { + const QString dir = core.attachmentDir(); + check(!dir.isEmpty() && dir.contains(QLatin1String("/attachments/")), + QStringLiteral("the agent names its image directory (%1)").arg(dir)); + QImage pic(32, 16, QImage::Format_RGB32); + pic.fill(Qt::darkGreen); + QByteArray png; + QBuffer buf(&png); + buf.open(QIODevice::WriteOnly); + pic.save(&buf, "PNG"); + char aerr[256] = ""; + cst_attachment_t *stored = cst_attach_import_bytes( + cst_agent_session_path(core.agent()), "probe.png", png.constData(), + static_cast(png.size()), aerr, sizeof(aerr)); + check(stored != nullptr, + QStringLiteral("a picture can be stored next to the session")); + if (stored) { + QSize got; + const QImage shrunk = + images->requestImage(QString::fromUtf8(stored->rel), + &got, QSize(8, 8)); + check(!shrunk.isNull() && shrunk.width() == 8 && shrunk.height() == 4, + QStringLiteral("the provider serves it, shrunk to what was asked " + "(%1x%2)") + .arg(shrunk.width()) + .arg(shrunk.height())); + check(images->requestImage(QStringLiteral("../escape.png"), &got, + QSize(64, 64)) + .isNull(), + QStringLiteral("a name outside the attachment directory is refused")); + check(images->requestImage(QStringLiteral("no-such-file.png"), &got, + QSize(64, 64)) + .isNull(), + QStringLiteral("a file that is not there is refused")); + cst_attach_free(stored); + } + } + + /* --- a tool that made a picture: its card shows it -------------------- */ + { + const int before = list->property("count").toInt(); + const int thumbsBefore = + countNamed(win->contentItem(), QStringLiteral("imageThumb")); + event4("engineToolEvent", QStringLiteral("open"), QStringLiteral("call-img"), + QStringLiteral("view_image"), QStringLiteral("{\"path\":\"shot.png\"}")); + const QString imgs = + QStringLiteral("img-fixture.png|shot.png|view_image|24x8"); + check(QMetaObject::invokeMethod(win, "engineToolImages", + Q_ARG(QVariant, QStringLiteral("call-img")), + Q_ARG(QVariant, imgs)), + QStringLiteral("images a tool made reach the view")); + QTest::qWait(150); + const QVariantMap card = rowAt(before); + check(card.value(QStringLiteral("images")) + .toString() + .contains(QLatin1String("img-fixture.png")) && + card.value(QStringLiteral("name")).toString() == + QLatin1String("view_image"), + QStringLiteral("the card of that call carries the image")); + check(countNamed(win->contentItem(), QStringLiteral("imageThumb")) > + thumbsBefore, + QStringLiteral("its thumbnail is in the card")); + maybeShot(win, QStringLiteral("-toolimage")); + } + /* --- a narrow window: dock -> drawer, and the button opens it --- */ resizeTo(win, 460, 900); check(docked && !docked->isVisible(), diff --git a/app/tools/selftest.c b/app/tools/selftest.c index 0a5e275..5746c82 100644 --- a/app/tools/selftest.c +++ b/app/tools/selftest.c @@ -9,6 +9,7 @@ #include "cst_agent.h" #include "cst_tools.h" +#include "os_compat.h" #include #include @@ -54,8 +55,14 @@ static int probe_missing(char *why, size_t why_sz) { } int main(void) { - char ws[1024]; - snprintf(ws, sizeof(ws), "/tmp/cst-app-selftest-%d", (int)getpid()); + /* The scratch workspace goes where this host can actually write: /tmp on a + * desktop, the app's own directory on Android (where /tmp is off limits, + * so TMPDIR is pointed at it -- see app/src/main.cpp). */ + char tmp[1024] = ""; + if (os_temp_dir(tmp, sizeof(tmp)) != 0 || !tmp[0]) + snprintf(tmp, sizeof(tmp), "%s", "/tmp"); + char ws[1200]; + snprintf(ws, sizeof(ws), "%s/cst-app-selftest-%d", tmp, (int)getpid()); #ifdef _WIN32 _mkdir(ws); /* the engine wants its workspace to exist */ #else diff --git a/docs/android.md b/docs/android.md index df3dab8..0ec7d4a 100644 --- a/docs/android.md +++ b/docs/android.md @@ -38,6 +38,95 @@ nix-shell -p cmake ninja curl android-tools --run ' `--run-tests` builds the whole `tests/` suite, pushes it and runs every binary on the phone, printing a pass/fail line per test. +## The APK (Qt front end) + +`scripts/build_apk_android.sh [--install] [--logs] [--clean]` configures with +the Qt for Android kit, builds the `apk` target (Gradle, no port opened) and +optionally installs it. Bits it needs, all overridable through the +environment: + +| variable | what it points at | +| --- | --- | +| `QT_ANDROID` | Qt for Android 6.11.2 (`aqt install-qt all_os android 6.11.2 android_arm64_v8a -O ~/Qt --archives qtbase qtdeclarative qtquickcontrols2`) | +| `QT_HOST` | the host Qt of the same version (`aqt install-qt linux desktop 6.11.2 linux_gcc_64 -O ~/Qt --archives qtbase qtdeclarative qtsvg`) | +| `ANDROID_SDK_ROOT` | SDK with `platforms/android-` and `build-tools/` | +| `ANDROID_NDK_HOME` | the NDK the engine was built with | +| `JAVA_HOME` | JDK 17+ for Gradle | + +### The SDK without sdkmanager + +`nixpkgs`'s full `androidsdk` pulls the emulator and system images (gigabytes); +the two components actually needed are cheap. The platform comes from the nix +store (unfree, so the license flag is required) and build-tools from Google's +own zip, laid out as the SDK `sdkmanager` would: + +```sh +PLATFORM=$(NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix build \ + --impure --no-link --print-out-paths --expr \ + 'builtins.elemAt (import {}).androidenv.androidPkgs.platforms 2') # android-35 +mkdir -p ~/android-sdk/{build-tools,platforms} +ln -s $PLATFORM/libexec/android-sdk/platforms/android-35 ~/android-sdk/platforms/ +``` + +`androiddeployqt` then picks `build-tools` and `platform-tools` from there. For +build-tools, `dl.google.com` is the only source: from here it hands out +~100 KB/s per connection, so the script downloads the zip in 16 ranged +requests (≈25x faster) and fixes the exec bits Python's `zipfile` drops. + +### Host Qt on NixOS + +The host Qt is built for a regular distribution: its tools (`qmlimportscanner` +first of all) look for `libgssapi_krb5.so.2`, `libglib-2.0.so.0`, `libz.so.1`, +`libfontconfig`, `libX11`, `libEGL`/`libGL`, `libzstd`, `libbrotlidec`, +`libdbus-1`, `libfreetype`, `libstdc++`, none of which exist in NixOS's search +paths. The tools carry `RUNPATH=$ORIGIN/../lib`, so symlinking those libraries +from the nix store into `/lib/` is enough: + +```sh +nix build --no-link --print-out-paths nixpkgs#krb5.lib nixpkgs#dbus.lib \ + nixpkgs#libglvnd nixpkgs#fontconfig.lib nixpkgs#freetype nixpkgs#glib.lib \ + nixpkgs#libx11 nixpkgs#libxkbcommon nixpkgs#zlib nixpkgs#zstd \ + nixpkgs#brotli.lib nixpkgs#stdenv.cc.cc.lib # then ln -s ~/Qt/6.11.2/gcc_64/lib/ +``` + +Two more things the script does on the way: + +* **GLESv2/EGL**: Qt's `FindGLESv2.cmake` only searches `CMAKE_FIND_ROOT_PATH` + prefixes (`/include`), but the NDK keeps the headers in + `/usr/include` -- without `-DGLESv2_INCLUDE_DIR`/`-DEGL_INCLUDE_DIR` + `find_package(Qt6 Gui)` fails with "dependency GLESv2 could not be found". +* **`-L$PREFIX/lib` on every link type**: the APK's payload is a *shared + module*, so `CMAKE_SHARED_LINKER_FLAGS`/`CMAKE_MODULE_LINKER_FLAGS` need the + prefix too, not just the EXE flags. + +### Gradle + +The generated project resolves plugins and dependencies from `google()` and +`mavenCentral()`, plus a Gradle distribution from `services.gradle.org`. From +here all three are slow (dl.google.com ~100 KB/s), so two local tweaks: + +* `~/.gradle/init.d/*.gradle` (machine-local, not in the repo) prepends the + Aliyun/Tencent mirrors for `pluginManagement`, `dependencyResolutionManagement` + and every `buildscript`. +* the wrapper in `/src/3rdparty/gradle/gradle/wrapper/` points at + the Tencent Gradle mirror (51 MB/s vs 100 KB/s). + +`scripts/gradle-android-init.gradle` (installed into `~/.gradle/init.d/` by the +build script) switches off release lint: `lintVitalAnalyzeRelease` resolves its +own tool jars straight from `dl.google.com`, bypassing every repository the +build configures, and fails the build after all the real work is done. + +### Signing + +`androiddeployqt` copies the **unsigned** Gradle output and signs it itself +(`QT_ANDROID_SIGN_APK` + the `QT_ANDROID_KEYSTORE_*` variables; a signed Gradle +output makes it fail with "Could not copy unsigned APK"). The script exports +those variables and creates the standard Android debug keystore +(`~/.android/debug.keystore`, alias `androiddebugkey`) on first use; +`CST_ANDROID_KEYSTORE` and `CST_ANDROID_KEY_ALIAS`/`_PASS` override it. +`apksigner` is a bash script with a `#!/bin/bash` shebang, which NixOS does not +have, so the script rewrites the shebang of the build-tools wrappers. + ## What is Android-specific * **API level 24** (`API_LEVEL=24`): bionic provides threads, `exec` and @@ -66,8 +155,13 @@ on the phone, printing a pass/fail line per test. * The whole suite runs on the device (`scripts/build_android.sh arm64-v8a --run-tests`): 36/36 after the diff test learned to tell toybox's `diff` from GNU diffutils. -* Still to come: the Qt front end packaged as an APK. nixpkgs has no Qt for - Android build, so that needs a Qt host kit plus the `android_arm64_v8a` kit - (qt.io or aqtinstall), a JDK and the Android SDK platform/build-tools; - `app/src/platform/android/` (JNI tools, owner `app:android`) and the Gradle - project under `app/android/` do not exist yet. +* The APK builds, installs and runs: the Qt front end links the engine + in-process (no port), `main.cpp` puts the workspace, config and tmp under + `/data/user/0/xyz.whcself.cstagent/files/`, and a turn does a real HTTPS + request from inside the ART process (with no API key the provider answers + 401 -- the engine, curl and TLS parts of the round trip are what that + shows). Rotation keeps the session; the sidebar appears in landscape. +* Still to come: the Android platform tools behind `app:android` + (`notification_send`, clipboard, share sheet, SAF file picker), the back + gesture, and the icon glyphs -- `ThemeIcon` finds no icon files on the phone + and draws empty boxes where the desktop shows Breeze icons. diff --git a/include/cst_agent.h b/include/cst_agent.h index baec867..ee7337f 100644 --- a/include/cst_agent.h +++ b/include/cst_agent.h @@ -334,6 +334,12 @@ typedef struct { /* Full tool output, fired right after execution (before it is persisted). */ void (*on_tool_result)(const ctl_tool_call_t *call, const char *out, size_t len, int exit_code, void *ud); + /* Images a tool produced (view_image, MCP image content), fired right after + * the call ran and before the result is persisted. Only a call that made at + * least one image is reported, so a front end can show it at once instead of + * reading the store. */ + void (*on_tool_images)(const ctl_tool_call_t *call, + const cst_attachment_t *list, int n, void *ud); void *userdata; } cst_agent_hooks_t; @@ -378,6 +384,12 @@ int cst_agent_pending_count(const cst_agent_t *a); /* One line " (, 12 KB)"; -1 when i is out of range. */ int cst_agent_pending_describe(const cst_agent_t *a, int i, char *out, size_t out_sz); +/* The stored name of pending attachment i ("img-1a2b3c4d.png"), or NULL when + * i is out of range. The file is already in the session store, so a front end + * can show the image before the message is sent. */ +const char *cst_agent_pending_rel(const cst_agent_t *a, int i); +/* 1 when pending attachment i is an image, 0 otherwise (or out of range). */ +int cst_agent_pending_is_image(const cst_agent_t *a, int i); void cst_agent_pending_clear(cst_agent_t *a); /* Latest todo_write list of this session, or NULL. */ const char *cst_agent_todo(const cst_agent_t *a); diff --git a/include/cst_version.h b/include/cst_version.h index 807fdc1..e694393 100644 --- a/include/cst_version.h +++ b/include/cst_version.h @@ -2,8 +2,8 @@ #define CST_VERSION_H #define CST_VERSION_MAJOR 0 -#define CST_VERSION_MINOR 8 -#define CST_VERSION_PATCH 15 +#define CST_VERSION_MINOR 9 +#define CST_VERSION_PATCH 0 #define CST_VERSION_STR_(x) #x #define CST_VERSION_STR(x) CST_VERSION_STR_(x) diff --git a/scripts/build_apk_android.sh b/scripts/build_apk_android.sh new file mode 100755 index 0000000..b6f72ed --- /dev/null +++ b/scripts/build_apk_android.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Build the Qt Quick Android APK of app/ (the engine is linked in-process, so +# the APK opens no port). This is the front end counterpart of +# scripts/build_android.sh, which cross-builds the engine alone. +# +# scripts/build_apk_android.sh [--install] [--logs] [--clean] +# +# --install adb install -r the debug APK on the connected device +# --logs stream logcat after installing (Qt + engine tags) +# --clean wipe the build directory first +# +# Needs, all overridable through the environment: +# QT_ANDROID Qt for Android (default ~/Qt//android_arm64_v8a) +# QT_HOST matching host Qt (default ~/Qt//gcc_64) +# ANDROID_SDK_ROOT SDK with platforms/ + build-tools/ (see docs/android.md) +# ANDROID_NDK_HOME NDK, same one as scripts/build_third_party_android.sh +# JAVA_HOME JDK 17+ for Gradle (falls back to the javac on PATH) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ABI="${ABI:-arm64-v8a}" +API_LEVEL="${API_LEVEL:-24}" +BUILD="$ROOT/build-apk-android" +PREFIX="$ROOT/third_party/prefix-android-$ABI" +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}" + +DO_INSTALL=0 +DO_LOGS=0 +for a in "$@"; do + case "$a" in + --install) DO_INSTALL=1 ;; + --logs) DO_INSTALL=1; DO_LOGS=1 ;; + --clean) rm -rf "$BUILD" ;; + *) echo "usage: $0 [--install] [--logs] [--clean]" >&2; exit 1 ;; + esac +done + +# --- Qt: Android target first, then the host tools of the same version ---- +if [[ -z "${QT_ANDROID:-}" ]]; then + # aqt names the directory after the ABI with underscores: android_arm64_v8a + for q in "$HOME"/Qt/*/android_"${ABI//-/_}"/lib/cmake/Qt6/qt.toolchain.cmake; do + [[ -f "$q" ]] && QT_ANDROID="${q%/lib/cmake/Qt6/qt.toolchain.cmake}" + done +fi +if [[ -z "${QT_ANDROID:-}" || ! -x "$QT_ANDROID/bin/qt-cmake" ]]; then + echo "set QT_ANDROID to a Qt for Android install ('aqt install-qt all_os android ...')" >&2 + exit 1 +fi +QT_VER="$(basename "$(dirname "$QT_ANDROID")")" +QT_HOST="${QT_HOST:-$(dirname "$QT_ANDROID")/gcc_64}" +if [[ ! -x "$QT_HOST/bin/qmake" && ! -x "$QT_HOST/bin/androiddeployqt" ]]; then + echo "set QT_HOST to the host Qt $QT_VER (same version, needed for androiddeployqt)" >&2 + exit 1 +fi + +NDK="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" +if [[ -z "$NDK" || ! -f "$NDK/build/cmake/android.toolchain.cmake" ]]; then + echo "set ANDROID_NDK_HOME to an NDK (see scripts/build_third_party_android.sh)" >&2 + exit 1 +fi +if [[ -z "${ANDROID_SDK_ROOT:-}" || ! -d "$ANDROID_SDK_ROOT/platforms" ]]; then + echo "set ANDROID_SDK_ROOT to an SDK with platforms/android- and build-tools/" >&2 + exit 1 +fi +if [[ -z "${JAVA_HOME:-}" ]]; then + JAVAC="$(command -v javac || true)" + [[ -n "$JAVAC" ]] && JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$JAVAC")")")" +fi +if [[ -z "${JAVA_HOME:-}" || ! -x "$JAVA_HOME/bin/java" ]]; then + echo "set JAVA_HOME to a JDK 17+ (Gradle needs it)" >&2 + exit 1 +fi +if [[ ! -f "$PREFIX/lib/libcurl.a" ]]; then + echo "missing $PREFIX/lib/libcurl.a -- run:" >&2 + echo " scripts/build_third_party_android.sh $ABI all" >&2 + exit 1 +fi +export ANDROID_SDK_ROOT ANDROID_NDK_HOME="$NDK" ANDROID_NDK_ROOT="$NDK" JAVA_HOME +export PATH="$JAVA_HOME/bin:$PATH" + +# androiddeployqt shells out to the generated project's gradlew, which picks up +# init scripts from $GRADLE_USER_HOME/init.d. Install ours (release lint off) +# so every build behaves the same. See the file for why lint has to go. +GRADLE_INIT_DIR="${GRADLE_USER_HOME:-$HOME/.gradle}/init.d" +mkdir -p "$GRADLE_INIT_DIR" +if ! cmp -s "$ROOT/scripts/gradle-android-init.gradle" "$GRADLE_INIT_DIR/cst-android.gradle"; then + install -m644 "$ROOT/scripts/gradle-android-init.gradle" "$GRADLE_INIT_DIR/cst-android.gradle" + echo "installed $GRADLE_INIT_DIR/cst-android.gradle" +fi + +# The release APK has to be signed to install. With CST_ANDROID_KEYSTORE the +# init script uses that keystore; otherwise it falls back to the standard +# Android debug key, created here on first use (same one Android Studio makes). +KEYSTORE="${CST_ANDROID_KEYSTORE:-$HOME/.android/debug.keystore}" +if [[ ! -f "$KEYSTORE" && -z "${CST_ANDROID_KEYSTORE:-}" ]]; then + mkdir -p "$(dirname "$KEYSTORE")" + keytool -genkeypair -keystore "$KEYSTORE" -storepass android -keypass android \ + -alias androiddebugkey -keyalg RSA -keysize 2048 -validity 10000 \ + -dname "CN=Android Debug,O=Android,C=US" >/dev/null + chmod 600 "$KEYSTORE" + echo "created debug keystore: $KEYSTORE" +fi +# androiddeployqt reads these (the path/alias/passwords never reach the CMake +# command line, only the QT_ANDROID_SIGN_APK flag does). +export QT_ANDROID_KEYSTORE_PATH="$KEYSTORE" +export QT_ANDROID_KEYSTORE_ALIAS="${CST_ANDROID_KEY_ALIAS:-androiddebugkey}" +export QT_ANDROID_KEYSTORE_STORE_PASS="${CST_ANDROID_KEYSTORE_PASS:-android}" +export QT_ANDROID_KEYSTORE_KEY_PASS="${CST_ANDROID_KEY_PASS:-android}" + +# NixOS has no /bin/bash, and build-tools' shell wrappers (apksigner is the one +# androiddeployqt calls, d8 and friends are the rest) start with #!/bin/bash. +# Point them at the bash that exists here; the SDK is a local copy, so this +# sticks. +BT_DIR="$(ls -d "$ANDROID_SDK_ROOT"/build-tools/* 2>/dev/null | sort -V | tail -1)" +BASH_BIN="$(command -v bash || echo /bin/sh)" +if [[ -n "$BT_DIR" ]]; then + while IFS= read -r f; do + sed -i "1s|^#!/bin/bash|#!$BASH_BIN|" "$f" && echo "fixed shebang: ${f#"$ANDROID_SDK_ROOT"/}" + done < <(grep -rls '^#!/bin/bash' "$BT_DIR" 2>/dev/null) +fi + +# Qt6Gui wants GLESv2/EGL: the NDK sysroot has both, but Qt's Find modules +# only search CMAKE_FIND_ROOT_PATH prefixes (/include), while the +# headers live in /usr/include -- point them there directly. +NDK_SYSROOT_INC="$NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include" +[[ -d "$NDK_SYSROOT_INC/GLES2" ]] || echo "warning: no GLES2 headers under $NDK_SYSROOT_INC" >&2 + +echo "Qt for Android : $QT_ANDROID ($QT_VER)" +echo "host Qt : $QT_HOST" +echo "SDK : $ANDROID_SDK_ROOT ($(ls "$ANDROID_SDK_ROOT/build-tools" | tr '\n' ' '))" +echo "NDK : $NDK" +echo "JDK : $JAVA_HOME" +echo "ABI : $ABI, minSdk $API_LEVEL" + +# The engine comes from libcstcore.a, exactly as on the desktop: no TUI, no +# HTTP server (src/webui_stub.c stands in), no bundled busybox inside an APK. +"$QT_ANDROID/bin/qt-cmake" -S "$ROOT" -B "$BUILD" -G Ninja \ + -DQT_HOST_PATH="$QT_HOST" \ + -DANDROID_ABI="$ABI" -DANDROID_PLATFORM="android-$API_LEVEL" \ + -DANDROID_SDK_ROOT="$ANDROID_SDK_ROOT" -DANDROID_NDK="$NDK" \ + -DGLESv2_INCLUDE_DIR="$NDK_SYSROOT_INC" -DEGL_INCLUDE_DIR="$NDK_SYSROOT_INC" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + -DCURL_INCLUDE_DIR="$PREFIX/include" -DCURL_LIBRARY="$PREFIX/lib/libcurl.a" \ + -DCMAKE_EXE_LINKER_FLAGS="-L$PREFIX/lib" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L$PREFIX/lib" \ + -DCMAKE_MODULE_LINKER_FLAGS="-L$PREFIX/lib" \ + -DCST_BUILD_TUI=OFF -DCST_BUILD_WEBUI=OFF -DCST_BUILD_APP=ON \ + -DCST_BUNDLE_BUSYBOX=OFF +cmake --build "$BUILD" --target apk -j"$JOBS" + +# AGP names the file after the variant and signing state +# (android-build-release.apk vs ...-unsigned.apk), so take the newest one. +APK="$(find "$BUILD/app/android-build/build/outputs/apk" "$BUILD" -name '*.apk' \ + -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)" +[[ -n "$APK" && -f "$APK" ]] || { echo "no APK produced" >&2; exit 1; } +echo +echo "APK: $APK ($(du -h "$APK" | cut -f1))" +[[ "$DO_INSTALL" == 1 ]] || exit 0 + +ADB="${ADB:-adb}" +command -v "$ADB" >/dev/null || { echo "adb not found (nix-shell -p android-tools)" >&2; exit 1; } +"$ADB" install -r "$APK" +echo "installed: $("$ADB" shell pm list packages | grep cstagent)" +[[ "$DO_LOGS" == 1 ]] || exit 0 +echo "--- am start (Ctrl-C to stop; logcat follows)" +"$ADB" shell monkey -p xyz.whcself.cstagent -c android.intent.category.LAUNCHER 1 >/dev/null +sleep 2 +exec "$ADB" logcat -s cst-agent:V Qt:V AndroidRuntime:E libc:V DEBUG:V diff --git a/scripts/gradle-android-init.gradle b/scripts/gradle-android-init.gradle new file mode 100644 index 0000000..cb515c4 --- /dev/null +++ b/scripts/gradle-android-init.gradle @@ -0,0 +1,25 @@ +// Gradle init script for the Android APK build, installed as +// ~/.gradle/init.d/cst-android.gradle by scripts/build_apk_android.sh. +// +// Qt's generated gradle project only sets "lintOptions { abortOnError false }", +// so the release build still runs lintVitalAnalyzeRelease -- and that task +// resolves its own tool jars (manifest-merger, lint-typedef-remover, ...) +// straight from dl.google.com, bypassing every repository the project (or an +// init script) configures. On a slow or filtered route that fails the build +// after all the real work is done, and nothing here wants lint output anyway. +gradle.afterProject { project -> + def android = project.extensions.findByName('android') + if (android != null) { + // AGP keeps both spellings around: lintOptions (deprecated) and lint. + try { android.lintOptions.checkReleaseBuilds = false } catch (Exception ignored) { } + try { android.lint.checkReleaseBuilds = false } catch (Exception ignored) { } + + // NOTE: no signing config here on purpose. androiddeployqt copies the + // *unsigned* gradle output and signs it itself (--sign); a signed + // gradle APK makes it fail with "Could not copy unsigned APK". The + // keystore is wired up from app/CMakeLists.txt through + // QT_ANDROID_SIGN_APK + the QT_ANDROID_KEYSTORE_* environment. + } + // Belt and braces: switch the tasks themselves off. + project.tasks.matching { it.name.startsWith('lintVital') }.configureEach { it.enabled = false } +} diff --git a/src/agent_loop.c b/src/agent_loop.c index 40eb0d3..7885f2c 100644 --- a/src/agent_loop.c +++ b/src/agent_loop.c @@ -1181,6 +1181,8 @@ static void exec_end(turn_t *t, pcall_t *pc, exec_t *x) { if (t->hooks && t->hooks->on_tool_result) t->hooks->on_tool_result(c, pc->output, pc->output_len, pc->exit_code, t->hooks->userdata); + if (t->hooks && t->hooks->on_tool_images && pc->nattach > 0) + t->hooks->on_tool_images(c, pc->attach, pc->nattach, t->hooks->userdata); if (pc->status != PC_DENIED) { char d[96]; snprintf(d, sizeof(d), "%sexit=%d bytes=%zu", @@ -2173,6 +2175,20 @@ int cst_agent_attach_bytes(cst_agent_t *a, const char *name, const char *data, err, err_sz); } +/* The stored name of a pending attachment: the window serves the file from the + * session's attachment directory while the message is still being written. */ +const char *cst_agent_pending_rel(const cst_agent_t *a, int i) { + if (!a || i < 0 || i >= a->npending) + return NULL; + return a->pending[i].rel; +} + +int cst_agent_pending_is_image(const cst_agent_t *a, int i) { + if (!a || i < 0 || i >= a->npending) + return 0; + return cst_attach_is_image(&a->pending[i]); +} + const char *cst_agent_todo(const cst_agent_t *a) { return a ? a->todo : NULL; } const char *cst_agent_notes(const cst_agent_t *a) { diff --git a/tests/test_attach.c b/tests/test_attach.c index a8aa971..df19c75 100644 --- a/tests/test_attach.c +++ b/tests/test_attach.c @@ -540,6 +540,15 @@ static void test_agent_pending(const char *root) { EXPECT(cst_agent_pending_describe(a, 0, line, sizeof(line)) == 0 && strstr(line, "4x2") != NULL, "pending describe shows the size"); + EXPECT(cst_agent_pending_is_image(a, 0) == 1, + "a pending image says that it is one"); + EXPECT(cst_agent_pending_rel(a, 0) != NULL && + cst_attach_rel_ok(cst_agent_pending_rel(a, 0)), + "its stored name is one a view may serve"); + EXPECT(cst_agent_pending_rel(a, -1) == NULL && + cst_agent_pending_rel(a, 99) == NULL && + cst_agent_pending_is_image(a, 99) == 0, + "out of range has no name and is no image"); EXPECT(cst_agent_attach_file(a, "/nonexistent/x.png", err, sizeof(err)) == -1, "missing file refused");