Hermit/app/src/ui/UiTest.cpp

/* --uitest: drive the window with synthesized input.
 *
 * The GUI is checked the way a person uses it -- a mouse click on a segment,
 * a finger tap on another, a narrowed window that has to collapse the
 * sidebar -- instead of only loading the QML and hoping. Qt's test event
 * synthesis does the pointing, so this runs on any platform, with or without
 * a touchscreen, and can be scripted (see app/README.md). */

#include "UiTest.h"

#include <QGuiApplication>
#include <QClipboard>
#include <QFile>
#include <QImage>
#include <QJsonDocument>
#include <QJsonObject>
#include <QPointingDevice>
#include <QQuickItem>
#include <QQmlApplicationEngine>
#include <QQmlContext>
#include <QQuickWindow>
#include <QTest>

#include <QVariantMap>

#include <cstdio>
#include <cstring>

#include "cst_agent.h"
#include "ui/ThemeIcons.h"

namespace app {
namespace {

int g_fail;

void check(bool ok, const QString &what) {
  std::printf("%s %s\n", ok ? "ok  " : "FAIL", qPrintable(what));
  std::fflush(stdout);
  if (!ok)
    g_fail++;
}

QQuickItem *byName(QQuickItem *root, const QString &name) {
  if (!root)
    return nullptr;
  if (root->objectName() == name)
    return root;
  const QList<QQuickItem *> kids = root->childItems();
  for (QQuickItem *c : kids)
    if (QQuickItem *hit = byName(c, name))
      return hit;
  return nullptr;
}

int countNamed(QQuickItem *root, const QString &prefix) {
  if (!root)
    return 0;
  int n = root->objectName().startsWith(prefix) ? 1 : 0;
  const QList<QQuickItem *> kids = root->childItems();
  for (QQuickItem *c : kids)
    n += countNamed(c, prefix);
  return n;
}

/* An item by objectName. The QObject tree is tried first (it holds the named
 * controls); a view's delegates can be owned by the model instead, so the
 * visual tree -- what a finger can actually reach -- is walked as well. */
QQuickItem *item(QQuickWindow *win, const QString &name) {
  if (QQuickItem *it = win->findChild<QQuickItem *>(name))
    return it;
  return byName(win->contentItem(), name);
}

/* The centre of an item, in window coordinates. */
QPoint centre(QQuickItem *it) {
  const QPointF p = it->mapToScene(QPointF(it->width() / 2, it->height() / 2));
  return p.toPoint();
}

void clickMouse(QQuickWindow *win, const QString &name) {
  QQuickItem *it = item(win, name);
  if (!it || !it->isVisible()) {
    check(false, QStringLiteral("mouse click %1: not there").arg(name));
    return;
  }
  QTest::mouseClick(win, Qt::LeftButton, Qt::NoModifier, centre(it));
}

void tapTouch(QQuickWindow *win, QPointingDevice *dev, const QString &name) {
  QQuickItem *it = item(win, name);
  if (!it || !it->isVisible()) {
    check(false, QStringLiteral("touch tap %1: not there").arg(name));
    return;
  }
  const QPoint p = centre(it);
  QTest::touchEvent(win, dev).press(0, p, win);
  QTest::touchEvent(win, dev).release(0, p, win);
}

void resizeTo(QQuickWindow *win, int w, int h) {
  win->resize(w, h);
  QTest::qWait(250); /* let the layouts settle */
}

/* A stored session with a tool call in it, written through the store's own
 * API, so the file is exactly what the engine reads back. */
QString writeFixtureSession(const QString &ws, const QString &id) {
  const QByteArray w = ws.toUtf8();
  char path[4096] = "";
  if (cst_agent_store_path_for_id(w.constData(), id.toUtf8().constData(), path,
                                  sizeof(path)) != 0)
    return QString();
  cst_session_t *s = cst_session_open(path);
  if (!s)
    return QString();
  cst_message_t m;
  memset(&m, 0, sizeof(m));
  m.role = CST_ROLE_USER;
  m.content = strdup("what is here?");
  cst_session_append(s, &m);
  memset(&m, 0, sizeof(m));
  m.role = CST_ROLE_ASSISTANT;
  m.content = strdup("let me look. ");
  m.tool_calls_json = strdup(
      "[{\"id\":\"call_1\",\"type\":\"function\",\"function\":"
      "{\"name\":\"read_file\",\"arguments\":\"{\\\"path\\\":"
      "\\\"README.md\\\"}\"}}]");
  cst_session_append(s, &m);
  memset(&m, 0, sizeof(m));
  m.role = CST_ROLE_TOOL;
  m.content = strdup("hello\n");
  snprintf(m.tool_call_id, sizeof(m.tool_call_id), "%s", "call_1");
  snprintf(m.tool_name, sizeof(m.tool_name), "%s", "read_file");
  cst_session_append(s, &m);
  memset(&m, 0, sizeof(m));
  m.role = CST_ROLE_ASSISTANT;
  m.reasoning = strdup("checked the file");
  m.content = strdup("a repo.");
  cst_session_append(s, &m);
  cst_session_close(s);
  return id;
}

/* A picture of an interesting state on request: the only way to look at the
 * layout on a machine without a screen (CST_APP_SHOT=shot.png makes the run
 * write shot.png at the end and shot-settings.png while the settings dialog
 * is open). */
void maybeShot(QQuickWindow *win, const QString &suffix) {
  if (!qEnvironmentVariableIsSet("CST_APP_SHOT"))
    return;
  const QString base = qEnvironmentVariable("CST_APP_SHOT");
  const QString path =
      suffix.isEmpty() ? base
      : base.endsWith(QLatin1String(".png"))
          ? base.left(base.size() - 4) + suffix + QStringLiteral(".png")
          : base + suffix + QStringLiteral(".png");
  std::printf("%s %s\n",
              win->grabWindow().save(path) ? "note wrote" : "note cannot write",
              qPrintable(path));
}

}  // namespace

int runUiTest(AgentCore &core, const QString &size) {
  int width = 1000, height = 720;
  const int x = size.indexOf(QLatin1Char('x'));
  if (x > 0) {
    const int w = size.left(x).toInt(), h = size.mid(x + 1).toInt();
    if (w > 0 && h > 0) {
      width = w;
      height = h;
    }
  }

  QQmlApplicationEngine engine;
  installThemeIcons(&engine); /* the window's icons, same as main() */
  engine.rootContext()->setContextProperty(QStringLiteral("agent"), &core);
  engine.rootContext()->setContextProperty(QStringLiteral("initialWidth"), width);
  engine.rootContext()->setContextProperty(QStringLiteral("initialHeight"), height);
  engine.load(QUrl(QStringLiteral("qrc:/qt/qml/CstAgent/Main.qml")));
  if (engine.rootObjects().isEmpty()) {
    std::printf("FAIL the window did not load\n");
    return 1;
  }
  QQuickWindow *win = qobject_cast<QQuickWindow *>(engine.rootObjects().value(0));
  if (!win) {
    std::printf("FAIL the root object is not a window\n");
    return 1;
  }
  QTest::qWait(250);

  QQuickItem *docked = item(win, QStringLiteral("dockedSidebar"));
  QQuickItem *navButton = item(win, QStringLiteral("navButton"));
  QQuickItem *navPanel = item(win, QStringLiteral("navPanel"));
  QQuickItem *composer = item(win, QStringLiteral("composerInput"));
  QPointingDevice *touch = QTest::createTouchDevice();

  /* --- a desktop window: the sidebar is docked, the drawer is gone --- */
  check(docked && docked->isVisible(), QStringLiteral("wide window: sidebar docked"));
  check(navButton && !navButton->isVisible(),
        QStringLiteral("wide window: no drawer button"));
  check(core.mode() == QLatin1String("coding"),
        QStringLiteral("engine starts in coding mode"));

  /* --- mouse: pick the other mode --- */
  clickMouse(win, QStringLiteral("modeGeneral"));
  check(core.mode() == QLatin1String("general"),
        QStringLiteral("mouse click on \"general\" reaches the engine"));
  check(item(win, QStringLiteral("modeGeneral"))->property("current").toBool(),
        QStringLiteral("mouse click marks \"general\" as current"));

  /* --- touch: tap the first mode back --- */
  tapTouch(win, touch, QStringLiteral("modeCoding"));
  check(core.mode() == QLatin1String("coding"),
        QStringLiteral("touch tap on \"coding\" reaches the engine"));

  /* --- mouse: the plan switch is a two-way control --- */
  clickMouse(win, QStringLiteral("planSwitch"));
  check(core.planMode(), QStringLiteral("plan switch turns plan mode on"));
  clickMouse(win, QStringLiteral("planSwitch"));
  check(!core.planMode(), QStringLiteral("plan switch turns it off again"));

  /* --- keyboard: typing in the composer arms Send --- */
  QQuickItem *send = item(win, QStringLiteral("sendButton"));
  if (composer && send) {
    composer->forceActiveFocus();
    /* QTest::keyClicks() is QWidget-only; a window takes one key at a time. */
    for (const QChar &c : QStringLiteral("hello"))
      QTest::keyClick(win, c.toLatin1());
    check(send->property("enabled").toBool(),
          QStringLiteral("typing arms the send button"));
    QTest::keyClick(win, Qt::Key_A, Qt::ControlModifier);
    QTest::keyClick(win, Qt::Key_Backspace);
    check(!send->property("enabled").toBool(),
          QStringLiteral("an empty composer disarms it"));
  } else {
    check(false, QStringLiteral("composer and send button are reachable"));
  }

  /* --- sessions: the sidebar is the store's face ------------------------ */
  QQuickItem *list = item(win, QStringLiteral("transcript"));
  const QString fixture =
      writeFixtureSession(core.workspace(), QStringLiteral("20260101-120000-fix"));
  check(!fixture.isEmpty() && list != nullptr,
        QStringLiteral("a session fixture with a tool call is in the store"));
  QMetaObject::invokeMethod(win, "reloadSessions");
  QTest::qWait(150);
  int rowsSeen = countNamed(win->contentItem(), QStringLiteral("sessionRow:"));
  if (qEnvironmentVariableIsSet("CST_APP_DUMP"))
    std::printf("note: %d session rows in the visual tree\n", rowsSeen);
  const int listed = win->property("sessionsView").toList().size();
  check(item(win, QStringLiteral("sessionRow:") + fixture) != nullptr,
        QStringLiteral("the stored session shows up in the sidebar (%1 rows, %2 listed)")
            .arg(rowsSeen)
            .arg(listed));
  check(core.sessions().size() >= 2,
        QStringLiteral("the store lists it next to the open one"));

  /* the fixture is: user, agent + call, result, thought, agent */
  clickMouse(win, QStringLiteral("sessionRow:") + fixture);
  check(core.sessionId() == fixture,
        QStringLiteral("a click on a row switches the engine"));
  check(list && list->property("count").toInt() == 5,
        QStringLiteral("the reopened transcript is replayed into the view"));
  {
    bool toolOk = false;
    const QVariantList rows = core.history();
    for (const QVariant &v : rows) {
      const QVariantMap r = v.toMap();
      if (r.value(QStringLiteral("kind")).toString() == QLatin1String("tool") &&
          r.value(QStringLiteral("name")).toString() == QLatin1String("read_file"))
        toolOk = r.value(QStringLiteral("complete")).toBool() &&
                 r.value(QStringLiteral("output")).toString() ==
                     QLatin1String("hello\n");
    }
    check(toolOk, QStringLiteral("a tool result is paired with its call"));
  }

  /* new sessions: the engine moves, the view empties, the store grows */
  const int before = core.sessions().size();
  clickMouse(win, QStringLiteral("newSessionButton"));
  const QString first = core.sessionId();
  check(first != fixture, QStringLiteral("New starts another session"));
  clickMouse(win, QStringLiteral("newSessionButton"));
  const QString second = core.sessionId();
  check(second != first, QStringLiteral("New starts another one each time"));
  check(core.sessions().size() == before + 2,
        QStringLiteral("both new sessions are in the store"));
  check(list && list->property("count").toInt() == 0,
        QStringLiteral("a fresh session starts with an empty view"));

  /* the x drops a stored session and leaves the open one alone */
  QTest::qWait(150);
  clickMouse(win, QStringLiteral("sessionDelete:") + first);
  check(core.sessions().size() == before + 1,
        QStringLiteral("the x drops a stored session"));
  check(core.sessionId() == second,
        QStringLiteral("deleting another session leaves the open one alone"));

  /* and back to the fixture, which still has its five rows */
  QTest::qWait(150);
  clickMouse(win, QStringLiteral("sessionRow:") + fixture);
  check(core.sessionId() == fixture && list && list->property("count").toInt() == 5,
        QStringLiteral("switching back replays it again"));

  /* --- provider, model, key: the settings dialog ------------------------ */
  check(core.providers().size() > 0 &&
            !core.providers().first().toMap().value(QStringLiteral("name"))
                 .toString()
                 .isEmpty(),
        QStringLiteral("the provider presets are readable"));
  check(core.useProvider(QStringLiteral("no-such-provider"), QString(), QString())
            .contains(QLatin1String("no such provider")),
        QStringLiteral("switching to an unknown provider is refused"));
  check(core.usage().contains(QStringLiteral("requests")),
        QStringLiteral("the token counters are readable"));

  clickMouse(win, QStringLiteral("settingsButton"));
  QQuickItem *panel = item(win, QStringLiteral("settingsPanel"));
  check(panel && panel->isVisible(), QStringLiteral("the gear opens the settings dialog"));
  check(countNamed(win->contentItem(), QStringLiteral("providerRow:")) > 0,
        QStringLiteral("the dialog lists the providers"));
  maybeShot(win, QStringLiteral("-settings"));
  QQuickItem *mfield = item(win, QStringLiteral("settingsModel"));
  if (mfield && panel) {
    mfield->setProperty("text", QStringLiteral("cst-test-model"));
    clickMouse(win, QStringLiteral("settingsApply"));
    check(core.model() == QLatin1String("cst-test-model"),
          QStringLiteral("the model picked in the dialog reaches the engine"));
    QQuickItem *note = item(win, QStringLiteral("settingsNote"));
    check(note && note->property("text").toString().contains(
                      QLatin1String("cst-test-model")),
          QStringLiteral("the dialog says what it is using now"));
    clickMouse(win, QStringLiteral("settingsClose"));
    check(!panel->isVisible(), QStringLiteral("Close puts the dialog away"));
  } else {
    check(false, QStringLiteral("the dialog has a model field"));
  }

  /* --- attachments: a file goes next to the session -------------------- */
  const QString att = core.workspace() + QStringLiteral("/note.txt");
  {
    QFile f(att);
    f.open(QIODevice::WriteOnly);
    f.write("hello from a fixture\n");
  }
  check(core.attachFile(att).isEmpty(),
        QStringLiteral("a file on disk can be attached"));
  check(core.pendingAttachments().size() == 1,
        QStringLiteral("the engine reports one attachment for the next prompt"));
  check(core.pendingAttachments().first().contains(QLatin1String("note.txt")),
        QStringLiteral("the attachment names the file it came from"));
  check(!core.attachFile(QStringLiteral("/no/such/file-here")).isEmpty(),
        QStringLiteral("a missing file is refused with a reason"));
  QTest::qWait(120);
  check(countNamed(win->contentItem(), QStringLiteral("attachChip")) == 1,
        QStringLiteral("the composer lists it"));
  maybeShot(win, QStringLiteral("-attach"));
  clickMouse(win, QStringLiteral("clearAttachments"));
  check(core.pendingAttachments().isEmpty(),
        QStringLiteral("clear all drops them"));

  /* the paste path: an image on the clipboard becomes an attachment */
  QGuiApplication *gui = qobject_cast<QGuiApplication *>(QCoreApplication::instance());
  if (gui && gui->clipboard()) {
    QImage img(8, 8, QImage::Format_RGB32);
    img.fill(Qt::red);
    gui->clipboard()->setImage(img);
    check(core.clipboardHasImage(), QStringLiteral("the clipboard image is seen"));
    check(core.attachClipboard().isEmpty() &&
              core.pendingAttachments().size() == 1,
          QStringLiteral("an image on the clipboard is attached"));
    core.clearPending();
  } else {
    check(false, QStringLiteral("there is a clipboard to paste from"));
  }

  /* --- tool calls: two calls of one tool are two cards ------------------ */
  const int rows0 = list ? list->property("count").toInt() : -1;
  auto rowAt = [&](int i) {
    QVariant v;
    QVariantMap r;
    if (!QMetaObject::invokeMethod(win, "rowJson", Q_RETURN_ARG(QVariant, v),
                                   Q_ARG(QVariant, i)))
      return r;
    return QJsonDocument::fromJson(v.toString().toUtf8()).object().toVariantMap();
  };
  auto event4 = [&](const char *fn, const QVariant &a, const QVariant &b,
                    const QVariant &c, const QVariant &d) {
    return QMetaObject::invokeMethod(win, fn, Q_ARG(QVariant, a),
                                     Q_ARG(QVariant, b), Q_ARG(QVariant, c),
                                     Q_ARG(QVariant, d));
  };
  check(event4("engineToolEvent", QStringLiteral("open"), QStringLiteral("call-1"),
               QStringLiteral("list_dir"), QStringLiteral("{\"path\":\"vendor\"}")) &&
            list->property("count").toInt() == rows0 + 1,
        QStringLiteral("a tool call opens a card"));
  event4("engineToolEvent", QStringLiteral("run"), QStringLiteral("call-1"),
         QStringLiteral("list_dir"), QString());
  /* the same tool a second time, while the first call is still open: pairing
     by name is what left the first card on "running" forever */
  event4("engineToolEvent", QStringLiteral("open"), QStringLiteral("call-2"),
         QStringLiteral("list_dir"), QStringLiteral("{\"path\":\"src\"}"));
  event4("engineToolResult", QStringLiteral("call-1"), QStringLiteral("list_dir"),
         QVariant(0), QStringLiteral("exit=0 bytes=532"));
  check(list->property("count").toInt() == rows0 + 2,
        QStringLiteral("two calls of one tool are two cards (%1 -> %2)")
            .arg(rows0)
            .arg(list->property("count").toInt()));
  {
    const QVariantMap first = rowAt(rows0), second = rowAt(rows0 + 1);
    check(first.value(QStringLiteral("state")).toString() == QLatin1String("done") &&
              first.value(QStringLiteral("output")).toString() ==
                  QLatin1String("exit=0 bytes=532"),
          QStringLiteral("the result lands on the card of its own call "
                         "(state '%1', output '%2', keys [%3])")
              .arg(first.value(QStringLiteral("state")).toString(),
                   first.value(QStringLiteral("output")).toString(),
                   first.keys().join(QLatin1Char(','))));
    check(second.value(QStringLiteral("state")).toString() == QLatin1String("pending") &&
              second.value(QStringLiteral("detail")).toString().contains(
                  QLatin1String("\"src\"")) &&
              second.value(QStringLiteral("output")).toString().isEmpty(),
          QStringLiteral("the other card is untouched (state '%1')")
              .arg(second.value(QStringLiteral("state")).toString()));
  }
  /* and a turn that ends with the call still open settles it */
  check(QMetaObject::invokeMethod(win, "engineTurnFinished", Q_ARG(QVariant, 0),
                                  Q_ARG(QVariant, QString())),
        QStringLiteral("the turn end is reachable too"));
  {
    const QVariantMap second = rowAt(rows0 + 1);
    check(second.value(QStringLiteral("complete")).toBool() &&
              second.value(QStringLiteral("state")).toString() ==
                  QLatin1String("unfinished"),
          QStringLiteral("an open call is settled when the turn ends"));
  }

  /* --- the answer is rendered while it streams -------------------------- */
  if (QMetaObject::invokeMethod(win, "engineAssistant", Q_ARG(QVariant, QStringLiteral("**bold** text")))) {
    const int last = list->property("count").toInt() - 1;
    QMetaObject::invokeMethod(win, "renderStreaming");
    const QVariantMap row = rowAt(last);
    check(row.value(QStringLiteral("kind")).toString() == QLatin1String("message") &&
              !row.value(QStringLiteral("complete")).toBool(),
          QStringLiteral("a delta opens a streaming answer"));
    check(row.value(QStringLiteral("html")).toString().contains(QLatin1Char('<')) &&
              row.value(QStringLiteral("text")).toString().contains(QLatin1String("bold")),
          QStringLiteral("markdown is rendered while the answer streams"));
  } else {
    check(false, QStringLiteral("assistant deltas are reachable"));
  }

  /* --- a narrow window: dock -> drawer, and the button opens it --- */
  resizeTo(win, 460, 900);
  check(docked && !docked->isVisible(),
        QStringLiteral("narrow window: the docked sidebar folds away"));
  check(navButton && navButton->isVisible(),
        QStringLiteral("narrow window: the drawer button appears"));
  clickMouse(win, QStringLiteral("navButton"));
  check(navPanel && navPanel->isVisible(),
        QStringLiteral("drawer opens over the transcript"));
  clickMouse(win, QStringLiteral("navButton"));
  check(navPanel && !navPanel->isVisible(),
        QStringLiteral("drawer closes again"));

  resizeTo(win, 1200, 820);
  check(docked && docked->isVisible(),
        QStringLiteral("widening again docks the sidebar back"));

  /* A picture of the last state on request (see maybeShot). */
  maybeShot(win, QString());

  std::printf("--- %s\n", g_fail ? "FAILED" : "all UI checks passed");
  return g_fail ? 1 : 0;
}

}  // namespace app