#include #include #include #include #include #include #include "cst_agent.h" #include "cst_attach.h" #include "cst_tools.h" #include "cst_util.h" #include "cst_web_md.h" #include "os_compat.h" #include "tools_internal.h" #define FETCH_DOWNLOAD_MAX (5 * 1024 * 1024) #define FETCH_DEFAULT_OUT (64 * 1024) #define FETCH_MAX_OUT (1024 * 1024) typedef struct { cst_buf_t body; int overflow; } fetch_t; static size_t on_body(char *p, size_t sz, size_t n, void *ud) { fetch_t *f = ud; size_t len = sz * n; if (f->body.len + len > FETCH_DOWNLOAD_MAX) { f->overflow = 1; return 0; /* abort */ } cst_buf_append(&f->body, p, len); return len; } static int tag_is(const char *p, const char *name) { size_t n = strlen(name); if (strncasecmp(p, name, n) != 0) return 0; char c = p[n]; return c == '>' || c == ' ' || c == '/' || c == '\t' || c == '\n' || c == '\r'; } static const char *const BLOCK_TAGS[] = { "p", "br", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "section", "article", "header", "footer", "pre", "table", "ul", "ol", "blockquote", "hr"}; static void put_entity(cst_buf_t *b, const char **pp, const char *end) { static const struct { const char *name; const char *rep; } ents[] = {{"&", "&"}, {"<", "<"}, {">", ">"}, {""", "\""}, {"'", "'"}, {"'", "'"}, {" ", " "}}; const char *p = *pp; for (size_t i = 0; i < sizeof(ents) / sizeof(ents[0]); i++) { size_t n = strlen(ents[i].name); if ((size_t)(end - p) >= n && strncmp(p, ents[i].name, n) == 0) { cst_buf_puts(b, ents[i].rep); *pp = p + n; return; } } if (end - p > 3 && p[1] == '#') { char *e = NULL; long cp = p[2] == 'x' || p[2] == 'X' ? strtol(p + 3, &e, 16) : strtol(p + 2, &e, 10); if (e && *e == ';' && cp > 0 && cp < 0x110000) { char u[4]; int k = 0; if (cp < 0x80) { u[k++] = (char)cp; } else if (cp < 0x800) { u[k++] = (char)(0xC0 | (cp >> 6)); u[k++] = (char)(0x80 | (cp & 0x3F)); } else if (cp < 0x10000) { u[k++] = (char)(0xE0 | (cp >> 12)); u[k++] = (char)(0x80 | ((cp >> 6) & 0x3F)); u[k++] = (char)(0x80 | (cp & 0x3F)); } else { u[k++] = (char)(0xF0 | (cp >> 18)); u[k++] = (char)(0x80 | ((cp >> 12) & 0x3F)); u[k++] = (char)(0x80 | ((cp >> 6) & 0x3F)); u[k++] = (char)(0x80 | (cp & 0x3F)); } cst_buf_append(b, u, (size_t)k); *pp = e + 1; return; } } cst_buf_append(b, p, 1); *pp = p + 1; } /* Crude HTML → text: drop script/style/head, tags become spaces or * newlines, entities decoded, blank runs collapsed. */ static char *html_to_text(const char *s, size_t n) { cst_buf_t b = {0}; const char *p = s, *end = s + n; for (const char *t = s; t + 7 < end; t++) { if (*t == '<' && strncasecmp(t + 1, "title", 5) == 0) { const char *gt = memchr(t, '>', (size_t)(end - t)); const char *e = gt ? gt + 1 : end; while (e + 7 < end && strncasecmp(e, " gt + 1) { cst_buf_puts(&b, "# "); cst_buf_append(&b, gt + 1, (size_t)(e - gt - 1)); cst_buf_puts(&b, "\n\n"); } break; } } while (p < end) { if (*p == '<') { const char *q = p + 1; int closing = *q == '/'; if (closing) q++; if (!closing && (tag_is(q, "script") || tag_is(q, "style") || tag_is(q, "head") || tag_is(q, "noscript"))) { const char *name = tag_is(q, "script") ? "= (long)nl && strncasecmp(e, name, nl) == 0)) e++; const char *gt = e < end ? memchr(e, '>', (size_t)(end - e)) : NULL; p = gt ? gt + 1 : end; continue; } if (end - p >= 4 && strncmp(p, "", 3) != 0) e++; p = e + 3 <= end ? e + 3 : end; continue; } int block = 0; for (size_t i = 0; i < sizeof(BLOCK_TAGS) / sizeof(BLOCK_TAGS[0]); i++) if (tag_is(q, BLOCK_TAGS[i])) block = 1; const char *gt = memchr(p, '>', (size_t)(end - p)); p = gt ? gt + 1 : end; cst_buf_puts(&b, block ? "\n" : " "); continue; } if (*p == '&') { put_entity(&b, &p, end); continue; } cst_buf_append(&b, p, 1); p++; } /* collapse whitespace: spaces within lines, at most one blank line */ cst_buf_t o = {0}; int nls = 0, sp = 0; for (size_t i = 0; i < b.len; i++) { char ch = b.buf[i]; if (ch == '\r') continue; if (ch == '\n') { if (nls < 2 && o.len) cst_buf_puts(&o, "\n"); nls++; sp = 0; continue; } if (ch == ' ' || ch == '\t') { sp = 1; continue; } if (sp && nls == 0 && o.len) cst_buf_puts(&o, " "); sp = 0; nls = 0; cst_buf_append(&o, &ch, 1); } cst_buf_free(&b); if (!o.buf) cst_buf_puts(&o, ""); return cst_buf_steal(&o); } static int textual(const char *ctype) { if (!ctype || !ctype[0]) return 1; return strncasecmp(ctype, "text/", 5) == 0 || strstr(ctype, "json") || strstr(ctype, "xml") || strstr(ctype, "javascript") || strstr(ctype, "yaml") || strstr(ctype, "toml"); } int cst_tool_web_fetch(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, cst_tool_result_t *r) { (void)ctx; const char *url = tool_arg(c, "url"); if (!url || !url[0]) return tool_set_result(r, "error: missing url", 1); while (*url == ' ') url++; if (strncasecmp(url, "http://", 7) != 0 && strncasecmp(url, "https://", 8) != 0) return tool_set_result(r, "error: only http:// and https:// URLs", 1); long max_out = tool_arg_long(c, "max_bytes", FETCH_DEFAULT_OUT); if (max_out <= 0) max_out = FETCH_DEFAULT_OUT; if (max_out > FETCH_MAX_OUT) max_out = FETCH_MAX_OUT; CURL *curl = curl_easy_init(); if (!curl) return tool_set_result(r, "error: curl init failed", 1); fetch_t f; memset(&f, 0, sizeof(f)); char errbuf[CURL_ERROR_SIZE]; errbuf[0] = 0; cst_curl_apply_ca(curl); curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L); #if LIBCURL_VERSION_NUM >= 0x075500 curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); #else curl_easy_setopt(curl, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS)); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS)); #endif curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "cst-agent/1.0 (+web_fetch)"); curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, on_body); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &f); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); CURLcode rc = curl_easy_perform(curl); long http = 0; char *ctype = NULL; char *final_url = NULL; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http); curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ctype); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &final_url); cst_buf_t b = {0}; int code = 0; if (rc != CURLE_OK && !f.overflow) { cst_buf_printf(&b, "error: fetch failed: %s", errbuf[0] ? errbuf : curl_easy_strerror(rc)); code = 1; } else { cst_buf_printf(&b, "HTTP %ld %s\ncontent-type: %s\n\n", http, final_url ? final_url : url, ctype ? ctype : "?"); if (http >= 400) code = 1; if (!textual(ctype)) { cst_buf_printf(&b, "(non-text content, %zu bytes not shown)\n", f.body.len); } else { char *text = NULL; size_t tlen = 0; int html = ctype && strstr(ctype, "html"); if (html) { text = html_to_text(f.body.buf ? f.body.buf : "", f.body.len); tlen = text ? strlen(text) : 0; } const char *src = html ? text : (f.body.buf ? f.body.buf : ""); size_t slen = html ? tlen : f.body.len; size_t keep = slen > (size_t)max_out ? cst_utf8_floor(src, (size_t)max_out) : slen; size_t clean_len = 0; char *clean = cst_utf8_sanitize(src, keep, &clean_len); if (clean) cst_buf_append(&b, clean, clean_len); free(clean); if (keep < slen || f.overflow) cst_buf_printf(&b, "\n...(truncated; %zu%s bytes in total)\n", slen, f.overflow ? "+" : ""); free(text); } } curl_easy_cleanup(curl); cst_buf_free(&f.body); return tool_take_buf(r, &b, code); } /* --- md_to_html ---------------------------------------------------------- */ #define MD_TO_HTML_MAX (8 * 1024 * 1024) /* The file name a URL ends in; "" when it names nothing (a bare host, a * directory). Kept to plain characters: it becomes a file name. */ static void name_from_url(const char *url, char *out, size_t n) { const char *p = strstr(url, "://"); p = p ? p + 3 : url; const char *q = strpbrk(p, "?#"); const char *end = q ? q : p + strlen(p); const char *slash = NULL; for (const char *s = p; s < end; s++) if (*s == '/') slash = s; const char *leaf = slash ? slash + 1 : p; size_t len = (size_t)(end - leaf); for (size_t i = 0; i < len; i++) { unsigned char ch = (unsigned char)leaf[i]; if (!(isalnum(ch) || ch == '.' || ch == '-' || ch == '_' || ch == '+')) { len = i; /* cut at the first byte that is not a file name's */ break; } } while (len && leaf[len - 1] == '.') len--; if (!len) snprintf(out, n, "download.bin"); else snprintf(out, n, "%.*s", (int)len, leaf); } /* "docs/notes.md" -> "notes" (the title a document falls back to). */ static void stem_of(const char *rel, char *out, size_t n) { char leaf[128]; snprintf(leaf, sizeof(leaf), "%.127s", rel && rel[0] ? rel : ""); char *slash = NULL; for (char *p = leaf; *p; p++) if (*p == '/' || *p == '\\') slash = p; char *base = slash ? slash + 1 : leaf; char *dot = strrchr(base, '.'); if (dot && dot != base) *dot = 0; snprintf(out, n, "%s", base[0] ? base : "document"); } /* A title turned into a file name: what a path cannot carry becomes '-', * control bytes go, and a name that is only dots or spaces is refused. */ static void stem_of_title(const char *title, char *out, size_t n) { size_t i = 0; for (const char *p = title; *p && i + 1 < n && i < 80; p++) { unsigned char ch = (unsigned char)*p; if (ch == '/' || ch == '\\' || ch == ':' || ch == '*' || ch == '?' || ch == '"' || ch == '<' || ch == '>' || ch == '|') ch = '-'; else if (ch < 0x20) continue; out[i++] = (char)ch; } while (i && (out[i - 1] == ' ' || out[i - 1] == '.')) i--; size_t start = 0; while (start < i && (out[start] == ' ' || out[start] == '.')) start++; i -= start; if (i) memmove(out, out + start, i); out[i] = 0; if (!i) snprintf(out, n, "document"); } /* The file md_to_html writes: `out` when given (a directory takes the * source's name inside it), otherwise the source file's stem, or the title. */ static void md_out_name(const ctl_tool_call_t *c, char *out_name, size_t n) { const char *file = tool_arg(c, "file"); const char *out_arg = tool_arg(c, "out"); const char *title = tool_arg(c, "title"); char stem[200]; if (file && file[0]) stem_of(file, stem, sizeof(stem)); else if (title && title[0]) stem_of_title(title, stem, sizeof(stem)); else snprintf(stem, sizeof(stem), "document"); if (out_arg && out_arg[0]) { snprintf(out_name, n, "%.300s", out_arg); size_t ol = strlen(out_name); if (ol && (out_name[ol - 1] == '/' || out_name[ol - 1] == '\\') && ol + strlen(stem) + 6 < n) snprintf(out_name + ol, n - ol, "%s.html", stem); } else { snprintf(out_name, n, "%s.html", stem); } } /* What the window shows before the write is approved. */ int cst_preview_md_to_html(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, char **out) { (void)ctx; const char *md = tool_arg(c, "markdown"); const char *file = tool_arg(c, "file"); if ((!md || !md[0]) && (!file || !file[0])) { *out = strdup("error: give markdown, or a .md file to render"); return -1; } char out_name[400]; md_out_name(c, out_name, sizeof(out_name)); cst_buf_t b = {0}; if (file && file[0]) cst_buf_printf(&b, "render %.200s → %.200s\n", file, out_name); else cst_buf_printf(&b, "render %zu bytes of markdown → %.200s\n", md ? strlen(md) : 0, out_name); *out = cst_buf_steal(&b); return 0; } int cst_tool_md_to_html(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, cst_tool_result_t *r) { const char *md = tool_arg(c, "markdown"); const char *file = tool_arg(c, "file"); if ((!md || !md[0]) && (!file || !file[0])) return tool_set_result(r, "error: give markdown, or a .md file to render", 1); const char *text = md; size_t len = md ? strlen(md) : 0; char *owned = NULL; if (file && file[0]) { char src[OS_PATH_MAX]; if (cst_sandbox_resolve(ctx->workspace_root, file, src, sizeof(src)) != 0) return tool_set_resultf(r, 1, "error: path outside workspace or missing " "parent: %.200s", file); size_t n = 0; owned = tool_slurp(src, &n, MD_TO_HTML_MAX); if (!owned) return tool_set_resultf(r, 1, "error: cannot read %.200s (or it is larger than " "%d MB)", file, MD_TO_HTML_MAX >> 20); text = owned; len = n; } char title[256], out_name[400]; const char *given = tool_arg(c, "title"); snprintf(title, sizeof(title), "%.200s", given && given[0] ? given : "Document"); md_out_name(c, out_name, sizeof(out_name)); char *html = cst_web_md_document(text, len, title); free(owned); if (!html) return tool_set_result(r, "error: cannot render the markdown", 1); tool_ensure_parent_dirs(ctx->workspace_root, out_name); char full[OS_PATH_MAX]; if (cst_sandbox_resolve(ctx->workspace_root, out_name, full, sizeof(full)) != 0) { free(html); return tool_set_resultf(r, 1, "error: path outside workspace: %.200s", out_name); } size_t bytes = strlen(html); int wrote = tool_write_whole(full, html, bytes); free(html); if (wrote != 0) return tool_set_resultf(r, 1, "error: cannot write %.200s", out_name); const char *session = ctx->agent ? cst_agent_session_path(ctx->agent) : NULL; cst_attachment_t *a = NULL; char err[256] = ""; if (session && session[0]) a = cst_attach_import_artifact(session, full, NULL, err, sizeof(err)); cst_buf_t out = {0}; cst_buf_printf(&out, "wrote %.200s (%zu bytes): one HTML file with everything " "inside it, ready to open in a browser\n", out_name, bytes); if (a) cst_buf_puts(&out, "(the user sees it as a file card they can open or share; " "you will not be shown its bytes again)\n"); else if (err[0]) cst_buf_printf(&out, "(the file is in the workspace; handing it over " "failed: %.150s)\n", err); int rc = tool_take_buf(r, &out, 0); if (a) { r->attach = a; r->nattach = 1; } return rc; } /* --- download ------------------------------------------------------------ */ #define DOWNLOAD_MAX (128 * 1024 * 1024) #define DOWNLOAD_TIMEOUT 300L typedef struct { FILE *fp; size_t written; size_t cap; int overflow; int failed; } dl_t; static size_t on_file(char *p, size_t sz, size_t n, void *ud) { dl_t *d = ud; size_t len = sz * n; if (d->written + len > d->cap) { d->overflow = 1; return 0; /* abort: the cap is what keeps a wrong URL from filling a disk */ } if (fwrite(p, 1, len, d->fp) != len) { d->failed = 1; return 0; } d->written += len; return len; } int cst_tool_download(const cst_tool_ctx_t *ctx, const ctl_tool_call_t *c, cst_tool_result_t *r) { const char *url = tool_arg(c, "url"); if (!url || !url[0]) return tool_set_result(r, "error: missing url", 1); while (*url == ' ') url++; if (strncasecmp(url, "http://", 7) != 0 && strncasecmp(url, "https://", 8) != 0) return tool_set_result(r, "error: only http:// and https:// URLs", 1); long cap = tool_arg_long(c, "max_bytes", DOWNLOAD_MAX); if (cap <= 0 || cap > DOWNLOAD_MAX) cap = DOWNLOAD_MAX; const char *path = tool_arg(c, "path"); char rel[400]; if (path && path[0]) snprintf(rel, sizeof(rel), "%s", path); else name_from_url(url, rel, sizeof(rel)); size_t rl = strlen(rel); if (rl && (rel[rl - 1] == '/' || rel[rl - 1] == '\\')) { /* the caller named the directory: the URL names the file */ char name[200]; name_from_url(url, name, sizeof(name)); snprintf(rel + rl, sizeof(rel) - rl, "%s", name); } tool_ensure_parent_dirs(ctx->workspace_root, rel); char full[OS_PATH_MAX]; if (cst_sandbox_resolve(ctx->workspace_root, rel, full, sizeof(full)) != 0) return tool_set_resultf(r, 1, "error: path outside workspace or missing parent: " "%.200s", rel); FILE *fp = fopen(full, "wb"); if (!fp) return tool_set_resultf(r, 1, "error: cannot write %.200s", rel); CURL *curl = curl_easy_init(); if (!curl) { fclose(fp); remove(full); return tool_set_result(r, "error: curl init failed", 1); } dl_t d; memset(&d, 0, sizeof(d)); d.fp = fp; d.cap = (size_t)cap; char errbuf[CURL_ERROR_SIZE]; errbuf[0] = 0; cst_curl_apply_ca(curl); curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L); #if LIBCURL_VERSION_NUM >= 0x075500 curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); #else curl_easy_setopt(curl, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS)); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS)); #endif curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, DOWNLOAD_TIMEOUT); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "cst-agent/1.0 (+download)"); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, on_file); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &d); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); CURLcode rc = curl_easy_perform(curl); long http = 0; char *ctype = NULL; char *final_url = NULL; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http); curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ctype); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &final_url); int closed = fclose(fp) != 0; curl_easy_cleanup(curl); if (rc != CURLE_OK || d.failed || closed || http >= 400) { remove(full); /* half a file is worse than none */ if (d.overflow) return tool_set_resultf(r, 1, "error: %.200s is larger than %ld bytes and " "nothing was saved (raise max_bytes, up to %d " "MB)", url, cap, DOWNLOAD_MAX >> 20); return tool_set_resultf(r, 1, "error: download failed: HTTP %ld %.200s", http, errbuf[0] ? errbuf : curl_easy_strerror(rc)); } cst_buf_t b = {0}; cst_buf_printf(&b, "saved %.200s (%s, %zu bytes) from HTTP %ld %.200s\n", rel, ctype ? ctype : "application/octet-stream", d.written, http, final_url ? final_url : url); cst_buf_puts(&b, "(the bytes are on disk, not in this answer; read_file or " "read_pdf it when you need to look inside)\n"); return tool_take_buf(r, &b, 0); }