v0.8.6: general mode, clipboard text, PDF text extraction

Working mode: --mode coding|general (CST_MODE, settings.json, /mode,
WebUI run settings). general swaps the default framing for a general
assistant one -- workflow, safety notes, tool wording (run_shell, the
subagents) -- and /status shows it; minimal (the paper profile) stays
byte-frozen. The prompt state carries the mode so a switch leaves a
system note in history.

Clipboard: clipboard_read / clipboard_write tools (16 MiB cap read,
default 64 KiB window; writing is a WRITE permission with a preview) on
pbcopy / wl-copy / xclip / pbpaste / wl-paste / xclip and the Win32
clipboard. In the TUI, /attach paste falls back to clipboard text when
there is no image on it.

PDF: a dependency-free text extractor (src/pdf_text.c): inflate,
Flate/ASCIIHex/ASCII85, PNG/TIFF predictors, layout-independent object
scan, object streams, page tree walk with inherited resources, content
lexer (Tj/TJ/'/" and the Td/TD/T*/Tm spacing heuristics), per-font
ToUnicode CMaps and inline images skipped. Encrypted files and pages
with no text layer are refused with the reason.

Attachments: /attach (a path or files dropped on the terminal) now take
images, PDFs and text. A PDF is stored as its text layer with the mime
still saying application/pdf and its page count; anything else has to be
UTF-8 text. Documents travel inline as text parts, so CST_NO_IMAGES and
text-only models no longer drop them, while images keep their old path.

Tests: test_pdf_text (plain, bare and zlib-framed Flate, inline image,
CID + ToUnicode, object stream, error paths), documents in test_attach,
inlined documents in test_llm_body, mode in test_config.

src/attach.c

@@ -4,6 +4,7 @@
 
 #include "cJSON.h"
 #include "cst_attach.h"
+#include "cst_pdf.h"
 #include "cst_util.h"
 #include "os_compat.h"
 
@@ -102,6 +103,9 @@ static int le16(const unsigned char *p) {
 }
 
 static const char *mime_of_ext(const char *mime) {
+  if (strcmp(mime, "application/pdf") == 0 ||
+      strcmp(mime, "text/plain") == 0)
+    return "txt"; /* a document is stored as the text it was reduced to */
   if (strcmp(mime, "image/png") == 0)
     return "png";
   if (strcmp(mime, "image/jpeg") == 0)
@@ -352,23 +356,73 @@ static void make_id(const char *name, const char *data, size_t n, char *out,
   snprintf(out, out_sz, "img-%08x", (unsigned)(h & 0xFFFFFFFFu));
 }
 
+/* n bytes, NUL-terminated: a text payload is handed around as a C string. */
+static char *memdup_z(const char *data, size_t n) {
+  char *copy = malloc(n + 1);
+  if (!copy)
+    return NULL;
+  memcpy(copy, data, n);
+  copy[n] = 0;
+  return copy;
+}
+
+/* What a picked file is, plus the payload to store for it: an image stays as
+ * it is, a PDF is reduced to its text layer (the mime still says PDF, the
+ * stored file holds text), anything else must be UTF-8 text. The caller frees
+ * the result; *out_len is its length. */
+static char *prepare(const char *data, size_t n, char *mime, size_t mime_sz,
+                     int *pages, size_t *out_len, char *err, size_t err_sz) {
+  *pages = 0;
+  *out_len = n;
+  if (cst_attach_sniff(data, n, mime, mime_sz) == 0) {
+    if (n > CST_ATTACH_MAX_BYTES) {
+      snprintf(err, err_sz, "image is %.1f MB; the limit is %d MB",
+               (double)n / (1024 * 1024), (int)(CST_ATTACH_MAX_BYTES >> 20));
+      return NULL;
+    }
+    char *copy = memdup_z(data, n);
+    if (!copy)
+      snprintf(err, err_sz, "out of memory");
+    return copy;
+  }
+  if (n >= 4 && memcmp(data, "%PDF", 4) == 0) {
+    int pg = 0;
+    char perr[256];
+    char *text = cst_pdf_text(data, n, &pg, perr, sizeof(perr));
+    if (!text) {
+      snprintf(err, err_sz, "cannot read the PDF: %.200s", perr);
+      return NULL;
+    }
+    snprintf(mime, mime_sz, "application/pdf");
+    *pages = pg;
+    *out_len = strlen(text);
+    return text;
+  }
+  if (n > CST_ATTACH_TEXT_MAX) {
+    snprintf(err, err_sz, "text is %.1f MB; the limit is %d MB",
+             (double)n / (1024 * 1024), (int)(CST_ATTACH_TEXT_MAX >> 20));
+    return NULL;
+  }
+  if (memchr(data, 0, n) || !cst_utf8_valid(data, n)) {
+    snprintf(err, err_sz,
+             "not an image (PNG, JPEG, GIF, WebP or BMP), a PDF or UTF-8 "
+             "text; the content decides, not the extension");
+    return NULL;
+  }
+  snprintf(mime, mime_sz, "text/plain");
+  char *copy = memdup_z(data, n);
+  if (!copy)
+    snprintf(err, err_sz, "out of memory");
+  return copy;
+}
+
+/* Write one payload whose kind is already known. */
 static cst_attachment_t *store(const char *session_path, const char *name,
-                               const char *data, size_t n, const char *origin,
-                               char *err, size_t err_sz) {
+                               const char *data, size_t n, const char *mime,
+                               int pages, const char *origin, char *err,
+                               size_t err_sz) {
   if (!name || !name[0])
     name = "image";
-  char mime[32];
-  if (cst_attach_sniff(data, n, mime, sizeof(mime)) != 0) {
-    snprintf(err, err_sz,
-             "not a supported image (PNG, JPEG, GIF, WebP or BMP; "
-             "checked by content, not by extension)");
-    return NULL;
-  }
-  if (n > CST_ATTACH_MAX_BYTES) {
-    snprintf(err, err_sz, "image is %.1f MB; the limit is %d MB",
-             (double)n / (1024 * 1024), (int)(CST_ATTACH_MAX_BYTES >> 20));
-    return NULL;
-  }
   char dir[OS_PATH_MAX];
   if (cst_attach_dir(session_path, dir, sizeof(dir)) != 0) {
     snprintf(err, err_sz, "cannot create the attachment directory");
@@ -386,7 +440,9 @@ static cst_attachment_t *store(const char *session_path, const char *name,
   snprintf(a->origin, sizeof(a->origin), "%s", origin ? origin : "user");
   snprintf(a->detail, sizeof(a->detail), "auto");
   a->bytes = (long long)n;
-  cst_attach_dimensions(data, n, &a->width, &a->height);
+  a->pages = pages;
+  if (cst_attach_is_image(a))
+    cst_attach_dimensions(data, n, &a->width, &a->height);
 
   char full[OS_PATH_MAX];
   if (cst_attach_full_path(session_path, a, full, sizeof(full)) != 0) {
@@ -419,14 +475,26 @@ cst_attachment_t *cst_attach_import_bytes(const char *session_path,
   }
   err[0] = 0;
   if (!session_path || !session_path[0]) {
-    snprintf(err, err_sz, "no session to store the image in");
+    snprintf(err, err_sz, "no session to store the file in");
     return NULL;
   }
   if (!data || !n) {
-    snprintf(err, err_sz, "image is empty");
+    snprintf(err, err_sz, "the file is empty");
     return NULL;
   }
-  return store(session_path, name, data, n, "user", err, err_sz);
+  char mime[32];
+  if (cst_attach_sniff(data, n, mime, sizeof(mime)) != 0) {
+    snprintf(err, err_sz,
+             "not a supported image (PNG, JPEG, GIF, WebP or BMP; "
+             "checked by content, not by extension)");
+    return NULL;
+  }
+  if (n > CST_ATTACH_MAX_BYTES) {
+    snprintf(err, err_sz, "image is %.1f MB; the limit is %d MB",
+             (double)n / (1024 * 1024), (int)(CST_ATTACH_MAX_BYTES >> 20));
+    return NULL;
+  }
+  return store(session_path, name, data, n, mime, 0, "user", err, err_sz);
 }
 
 cst_attachment_t *cst_attach_import_file(const char *session_path,
@@ -449,11 +517,30 @@ cst_attachment_t *cst_attach_import_file(const char *session_path,
   for (const char *p = src_path; *p; p++)
     if (*p == '/' || *p == '\\')
       leaf = p + 1;
-  cst_attachment_t *a = store(session_path, leaf, data, n, "user", err, err_sz);
+  char mime[32];
+  int pages = 0;
+  size_t pn = 0;
+  char *payload =
+      prepare(data, n, mime, sizeof(mime), &pages, &pn, err, err_sz);
   free(data);
+  if (!payload)
+    return NULL;
+  cst_attachment_t *a =
+      store(session_path, leaf, payload, pn, mime, pages, "user", err, err_sz);
+  free(payload);
   return a;
 }
 
+/* Images are opaque and travel as image parts; everything else is text the
+ * store holds and the request inlines. */
+int cst_attach_is_image(const cst_attachment_t *a) {
+  return a && strncmp(a->mime, "image/", 6) == 0;
+}
+
+int cst_attach_is_text(const cst_attachment_t *a) {
+  return a && !cst_attach_is_image(a);
+}
+
 char *cst_attach_read(const char *session_path, const cst_attachment_t *a,
                       size_t *out_len, char *err, size_t err_sz) {
   char dummy[8];
@@ -470,11 +557,12 @@ char *cst_attach_read(const char *session_path, const cst_attachment_t *a,
   char *data = read_whole(full, out_len, CST_ATTACH_MAX_BYTES);
   if (!data) {
     snprintf(err, err_sz,
-             "%.200s is missing; the image was deleted from the session "
-             "store",
+             "%.200s is missing; it was deleted from the session store",
              a->name);
     return NULL;
   }
+  if (!cst_attach_is_image(a)) /* a document is text, nothing to verify */
+    return data;
   char mime[32];
   if (cst_attach_sniff(data, *out_len, mime, sizeof(mime)) != 0) {
     snprintf(err, err_sz, "%.200s in the session store is no longer an image",
@@ -486,6 +574,8 @@ char *cst_attach_read(const char *session_path, const cst_attachment_t *a,
 }
 
 char *cst_attach_data_url(const char *session_path, const cst_attachment_t *a) {
+  if (!cst_attach_is_image(a)) /* a document travels as text, not as a URL */
+    return NULL;
   char err[256];
   size_t n = 0;
   char *data = cst_attach_read(session_path, a, &n, err, sizeof(err));
@@ -509,10 +599,21 @@ void cst_attach_describe(const cst_attachment_t *a, char *out, size_t out_sz) {
     snprintf(out, out_sz, "(none)");
     return;
   }
-  const char *kind = strncmp(a->mime, "image/", 6) == 0 ? a->mime + 6 : a->mime;
+  const char *kind = a->mime;
   char dim[32] = "";
-  if (a->width > 0 && a->height > 0)
-    snprintf(dim, sizeof(dim), "%dx%d ", a->width, a->height);
+  if (cst_attach_is_image(a)) {
+    if (strncmp(a->mime, "image/", 6) == 0)
+      kind = a->mime + 6;
+    if (a->width > 0 && a->height > 0)
+      snprintf(dim, sizeof(dim), "%dx%d ", a->width, a->height);
+  } else if (strcmp(a->mime, "application/pdf") == 0) {
+    kind = "pdf";
+    if (a->pages > 0)
+      snprintf(dim, sizeof(dim), "%d page%s ", a->pages,
+               a->pages == 1 ? "" : "s");
+  } else if (strcmp(a->mime, "text/plain") == 0 || !a->mime[0]) {
+    kind = "text";
+  }
   out[0] = 0;
   if (a->bytes >= 1024 * 1024)
     snprintf(out, out_sz, "%s%s %.1f MB", dim, kind,
@@ -580,6 +681,8 @@ char *cst_attachments_to_json(const cst_attachment_t *list, int n) {
       cJSON_AddNumberToObject(o, "width", a->width);
       cJSON_AddNumberToObject(o, "height", a->height);
     }
+    if (a->pages > 0)
+      cJSON_AddNumberToObject(o, "pages", a->pages);
     if (a->detail[0] && strcmp(a->detail, "auto") != 0)
       cJSON_AddStringToObject(o, "detail", a->detail);
     if (a->origin[0])
@@ -628,6 +731,8 @@ cst_attachment_t *cst_attachments_from_json(const void *array_json, int *n_out)
     a->width = cJSON_IsNumber(v) ? v->valueint : 0;
     v = cJSON_GetObjectItemCaseSensitive(it, "height");
     a->height = cJSON_IsNumber(v) ? v->valueint : 0;
+    v = cJSON_GetObjectItemCaseSensitive(it, "pages");
+    a->pages = cJSON_IsNumber(v) ? v->valueint : 0;
     v = cJSON_GetObjectItemCaseSensitive(it, "detail");
     snprintf(a->detail, sizeof(a->detail), "%s",
              cJSON_IsString(v) ? v->valuestring : "auto");