WHC/Hermit
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat: add persistent multi-model workers and orchestration budgets

Browse Source
This commit is contained in:
WHC
2026-10-19 11:29:26 +08:00
parent 235652604d
commit 143dba203b
14 changed files with 1043 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llm_openai.c
+12
View File
@@ -841,6 +841,8 @@ static void handle_payload(sse_t *s, const char *payload) {
s->st->prompt_tokens = (long)pt->valuedouble;
if (cJSON_IsNumber(ct))
s->st->completion_tokens = (long)ct->valuedouble;
if (cJSON_IsNumber(hit) || cJSON_IsNumber(cached) || cJSON_IsNumber(aread))
s->st->cache_reported = 1;
if (cJSON_IsNumber(hit))
s->st->cached_tokens = (long)hit->valuedouble;
else if (cJSON_IsNumber(cached))
@@ -1134,6 +1136,11 @@ int cst_llm_chat_stream(const cst_llm_request_t *req, char *err,
int ret = 0;
if (rep_dir && rep_dir[0]) {
if (req->request_gate && !req->request_gate(req->request_gate_ud)) {
snprintf(err, err_sz, "worker request budget exhausted");
ret = -1;
goto finish;
}
ret = replay_run(&s, rep_dir, seq, err, err_sz);
goto finish;
}
@@ -1150,6 +1157,11 @@ int cst_llm_chat_stream(const cst_llm_request_t *req, char *err,
}
for (int attempt = 0;; attempt++) {
if (req->request_gate && !req->request_gate(req->request_gate_ud)) {
snprintf(err, err_sz, "worker request budget exhausted");
ret = -1;
break;
}
CURL *curl = curl_easy_init();
if (!curl) {
snprintf(err, err_sz, "curl init failed");