Hermit/tests/test_models.c

#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE
#endif

#include "cst_agent.h"
#include "cst_models.h"
#include "os_compat.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define CST_TEST_SOCKETS
#include "cst_test_util.h"

#include <pthread.h>

#define CHECK(c)                                                               \
  do {                                                                         \
    if (!(c)) {                                                                \
      fprintf(stderr, "%s:%d: CHECK(%s) failed\n", __FILE__, __LINE__, #c);   \
      exit(1);                                                                 \
    }                                                                          \
  } while (0)

/* One canned HTTP response per accepted connection. */
static const char *g_responses[8];
static int g_nresp;
static test_sock_t g_listen = TEST_BAD_SOCK;
static char g_request[4096];

static void *serve(void *arg) {
  (void)arg;
  for (int i = 0; i < g_nresp; i++) {
    test_sock_t c = accept(g_listen, NULL, NULL);
    if (c == TEST_BAD_SOCK)
      return NULL;
    long n = recv(c, g_request, sizeof(g_request) - 1, 0);
    g_request[n > 0 ? n : 0] = 0;
    const char *r = g_responses[i];
    long off = 0, len = (long)strlen(r);
    while (off < len) {
      long w = send(c, r + off, (int)(len - off), 0);
      if (w <= 0)
        break;
      off += w;
    }
    test_sock_close(c);
  }
  return NULL;
}

static int start_server(void) {
  test_sock_init();
  g_listen = socket(AF_INET, SOCK_STREAM, 0);
  CHECK(g_listen != TEST_BAD_SOCK);
#ifndef _WIN32
  int one = 1;
  setsockopt(g_listen, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
#endif
  struct sockaddr_in a;
  memset(&a, 0, sizeof(a));
  a.sin_family = AF_INET;
  a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
  CHECK(bind(g_listen, (struct sockaddr *)&a, sizeof(a)) == 0);
  CHECK(listen(g_listen, 8) == 0);
  socklen_t al = sizeof(a);
  CHECK(getsockname(g_listen, (struct sockaddr *)&a, &al) == 0);
  return ntohs(a.sin_port);
}

#define RESP(status, body)                                                     \
  "HTTP/1.1 " status "\r\nContent-Type: application/json\r\n"                  \
  "Connection: close\r\n\r\n" body

int main(void) {
  char cfgdir[1024];
  CHECK(test_tmpdir(cfgdir, sizeof(cfgdir), "cst-models-cfg") == 0);
  setenv("XDG_CONFIG_HOME", cfgdir, 1); /* ignore a real models-cache */
  const cst_provider_t *tr = cst_provider_find("teamorouter");
  CHECK(tr && strcmp(tr->api_base, "https://api.teamorouter.com/v1") == 0);
  CHECK(tr->model[0] == 0); /* picked from GET /models */
  CHECK(cst_model_context_tokens("anthropic/claude-sonnet-4") == 200000);
  CHECK(cst_model_context_tokens("deepseek-flash") == 1000000);
  CHECK(cst_model_context_tokens("deepseek/deepseek-v4-pro") == 1000000);
  CHECK(cst_model_context_tokens("google/gemini-2.5-pro") == 1048576);
  CHECK(cst_model_context_tokens("unknown-model") == 128000);

  cst_model_caps_t caps;
  cst_model_caps("deepseek", "https://api.deepseek.com/v1", "deepseek-flash",
                 &caps);
  CHECK(caps.reasoning_passback && caps.prefix_completion);
  CHECK(caps.cache_style == CST_CACHE_AUTO);
  cst_model_caps("teamorouter", "https://api.teamorouter.com/v1",
                 "deepseek-flash", &caps);
  CHECK(!caps.reasoning_passback && !caps.prefix_completion);
  cst_model_caps("teamorouter", NULL, "anthropic/claude-sonnet-4", &caps);
  CHECK(caps.cache_style == CST_CACHE_ANTHROPIC);
  cst_model_caps("openai", NULL, "gpt-4.1", &caps);
  CHECK(caps.cache_style == CST_CACHE_OPENAI);
  cst_model_caps("openai", NULL, "o3-mini", &caps);
  CHECK(caps.cache_style == CST_CACHE_OPENAI);
  cst_model_caps("minimax", NULL, "MiniMax-M2", &caps);
  CHECK(caps.cache_style == CST_CACHE_AUTO);

  /* Beijing time: Thu 10:00 peak, Thu 13:00 and Sat 10:00 off-peak */
  CHECK(cst_price_is_peak((time_t)1791424800));
  CHECK(!cst_price_is_peak((time_t)1791435600));
  CHECK(!cst_price_is_peak((time_t)1791597600));
  cst_price_t pr;
  CHECK(cst_model_price("deepseek-flash", &pr) == 0);
  CHECK(pr.offpeak_half && pr.hit < pr.miss);
  double peak = cst_price_cost(&pr, 1000000, 1000000, 0, 1000000,
                               (time_t)1791424800);
  CHECK(peak > 9.99 && peak < 10.05); /* 2 + 0.04 + 8 */
  double off = cst_price_cost(&pr, 1000000, 1000000, 0, 1000000,
                              (time_t)1791435600);
  CHECK(off > peak / 2 - 0.001 && off < peak / 2 + 0.001);
  CHECK(cst_model_price("anthropic/claude-sonnet-4", &pr) == 0);
  CHECK(pr.write > pr.miss && strcmp(pr.currency, "$") == 0);
  CHECK(cst_model_price("MiniMax-M2", &pr) != 0);

  g_responses[0] = RESP("200 OK", "{\"object\":\"list\",\"data\":["
                                  "{\"id\":\"deepseek-flash\"},"
                                  "{\"id\":\"deepseek-v4-pro\"},"
                                  "{\"id\":\"deepseek-flash\"}]}");
  g_responses[1] = RESP("401 Unauthorized", "{\"error\":\"bad key\"}");
  g_responses[2] = RESP("200 OK", "<html>not json</html>");
  g_responses[3] = RESP("200 OK", "{\"models\":[{\"name\":\"m1\"},"
                                  "{\"name\":\"m2\"}]}");
  g_responses[4] = RESP("500 Internal Server Error", "{}");
  /* OpenRouter-style metadata; prices are USD per token, as strings */
  g_responses[5] = RESP(
      "200 OK",
      "{\"data\":[{\"id\":\"anthropic/claude-sonnet-4\","
      "\"context_length\":1000000,"
      "\"architecture\":{\"input_modalities\":[\"text\",\"image\"]},"
      "\"pricing\":{\"prompt\":\"0.000003\",\"completion\":\"0.000015\","
      "\"input_cache_read\":\"0.0000003\"}},"
      "{\"id\":\"old/text-model\","
      "\"architecture\":{\"modality\":\"text->text\"},"
      "\"pricing\":{\"prompt\":\"-1\",\"completion\":\"-1\"}},"
      "{\"id\":\"vllm-model\",\"max_model_len\":32768,"
      "\"modalities\":{\"input\":[\"text\"]}},"
      "{\"id\":\"plain\"}]}");
  g_nresp = 6;
  int port = start_server();
  pthread_t th;
  CHECK(pthread_create(&th, NULL, serve, NULL) == 0);

  char base[64];
  snprintf(base, sizeof(base), "http://127.0.0.1:%d/v1/", port);
  char ids[8][128], err[256];

  /* data[].id, duplicates folded, trailing slash on the base */
  int n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err));
  CHECK(n == 2);
  CHECK(strcmp(ids[0], "deepseek-flash") == 0);
  CHECK(strcmp(ids[1], "deepseek-v4-pro") == 0);
  CHECK(strncmp(g_request, "GET /v1/models ", 15) == 0);
  CHECK(strstr(g_request, "Authorization: Bearer sk-test") != NULL);

  n = cst_llm_list_models(base, "sk-bad", ids, 8, err, sizeof(err));
  CHECK(n == CST_MODELS_AUTH);
  CHECK(strstr(err, "401") != NULL);

  n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err));
  CHECK(n == -1);
  CHECK(err[0]);

  n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err));
  CHECK(n == 2 && strcmp(ids[1], "m2") == 0);

  n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err));
  CHECK(n == -1 && strstr(err, "500") != NULL);

  cst_model_meta_t meta[8];
  n = cst_llm_list_models_meta(base, "sk-test", meta, 8, err, sizeof(err));
  CHECK(n == 4);
  CHECK(meta[0].context == 1000000 && meta[0].vision == 1);
  CHECK(meta[0].has_price && meta[0].miss > 2.99 && meta[0].miss < 3.01 &&
        meta[0].out > 14.99 && meta[0].out < 15.01 && meta[0].hit < 0.31 &&
        meta[0].write == meta[0].miss);
  CHECK(meta[1].vision == 0 && !meta[1].has_price && meta[1].context == 0);
  CHECK(meta[2].context == 32768 && meta[2].vision == 0);
  CHECK(meta[3].vision == -1 && meta[3].context == 0 && !meta[3].has_price);

  pthread_join(th, NULL);
  test_sock_close(g_listen);

  /* nothing listening: network error, not an auth failure */
  n = cst_llm_list_models(base, "sk-test", ids, 8, err, sizeof(err));
  CHECK(n == -1);

  test_rmrf(cfgdir);
  puts("test_models: ok");
  return 0;
}