WHC
e0b0eddf3a
Working mode: --mode coding|general (CST_MODE, settings.json, /mode, WebUI run settings). general swaps the default framing for a general assistant one -- workflow, safety notes, tool wording (run_shell, the subagents) -- and /status shows it; minimal (the paper profile) stays byte-frozen. The prompt state carries the mode so a switch leaves a system note in history. Clipboard: clipboard_read / clipboard_write tools (16 MiB cap read, default 64 KiB window; writing is a WRITE permission with a preview) on pbcopy / wl-copy / xclip / pbpaste / wl-paste / xclip and the Win32 clipboard. In the TUI, /attach paste falls back to clipboard text when there is no image on it. PDF: a dependency-free text extractor (src/pdf_text.c): inflate, Flate/ASCIIHex/ASCII85, PNG/TIFF predictors, layout-independent object scan, object streams, page tree walk with inherited resources, content lexer (Tj/TJ/'/" and the Td/TD/T*/Tm spacing heuristics), per-font ToUnicode CMaps and inline images skipped. Encrypted files and pages with no text layer are refused with the reason. Attachments: /attach (a path or files dropped on the terminal) now take images, PDFs and text. A PDF is stored as its text layer with the mime still saying application/pdf and its page count; anything else has to be UTF-8 text. Documents travel inline as text parts, so CST_NO_IMAGES and text-only models no longer drop them, while images keep their old path. Tests: test_pdf_text (plain, bare and zlib-framed Flate, inline image, CID + ToUnicode, object stream, error paths), documents in test_attach, inlined documents in test_llm_body, mode in test_config.
251 lines
10 KiB
C
251 lines
10 KiB
C
/* Text extraction from PDFs: hand-made fixtures for the shapes that matter
|
|
* (plain content, FlateDecode, CID fonts with ToUnicode CMaps, object
|
|
* streams) plus the two error paths (encrypted, not a PDF). */
|
|
|
|
#include "cst_pdf.h"
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
static int failures = 0;
|
|
|
|
#define CHECK(cond, what) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, what); \
|
|
failures++; \
|
|
} \
|
|
} while (0)
|
|
|
|
static char *extract(const char *pdf, int *pages, char *err, size_t err_sz) {
|
|
return cst_pdf_text(pdf, strlen(pdf), pages, err, err_sz);
|
|
}
|
|
|
|
static int has(const char *hay, const char *needle) {
|
|
return hay && strstr(hay, needle) != NULL;
|
|
}
|
|
|
|
/* --- 1. plain, uncompressed content, two lines via Td --------------- */
|
|
|
|
static const char PLAIN[] =
|
|
"%PDF-1.4\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
|
|
"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
|
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n"
|
|
"5 0 obj\n<< >>\nstream\n"
|
|
"BT\n/F1 12 Tf\n72 720 Td\n(Line one) Tj\n0 -14 Td\n"
|
|
"(A \\(paren\\) and a \\\\ backslash) Tj\nET\n"
|
|
"endstream\nendobj\n"
|
|
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
|
|
|
static void test_plain(void) {
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = extract(PLAIN, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(pages == 1, "one page");
|
|
CHECK(has(t, "Line one"), "first line");
|
|
CHECK(has(t, "A (paren) and a \\ backslash"), "escapes");
|
|
CHECK(!has(t, "\r"), "no CR left");
|
|
free(t);
|
|
}
|
|
|
|
/* --- 2. FlateDecode content stream ---------------------------------- */
|
|
|
|
/* raw deflate (zlib, level 9, no header) of
|
|
* "BT /F1 12 Tf 72 720 Td (Hello, compressed world) Tj ET\n" */
|
|
static const unsigned char FLATE[] = {
|
|
0x73, 0x0a, 0x51, 0xd0, 0x77, 0x33, 0x54, 0x30, 0x34, 0x52, 0x08, 0x49,
|
|
0x53, 0x30, 0x37, 0x02, 0x22, 0x03, 0x85, 0x90, 0x14, 0x05, 0x0d, 0x8f,
|
|
0xd4, 0x9c, 0x9c, 0x7c, 0x1d, 0x85, 0xe4, 0xfc, 0xdc, 0x82, 0xa2, 0xd4,
|
|
0xe2, 0xe2, 0xd4, 0x14, 0x85, 0xf2, 0xfc, 0xa2, 0x9c, 0x14, 0x4d, 0x85,
|
|
0x90, 0x2c, 0x05, 0xd7, 0x10, 0x2e, 0x00};
|
|
|
|
static void test_flate(void) {
|
|
char pdf[2048];
|
|
int n = snprintf(pdf, sizeof(pdf),
|
|
"%%PDF-1.4\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font "
|
|
"<< /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
|
"4 0 obj\n<< /Type /Font /Subtype /Type1 >>\nendobj\n"
|
|
"5 0 obj\n<< /Filter /FlateDecode >>\nstream\n");
|
|
memcpy(pdf + n, FLATE, sizeof(FLATE));
|
|
n += (int)sizeof(FLATE);
|
|
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
|
"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(has(t, "Hello, compressed world"), "inflated text");
|
|
free(t);
|
|
}
|
|
|
|
/* 2b. zlib-framed FlateDecode (RFC 1950), which is what real writers emit:
|
|
* zlib.compress(b"BT /F1 12 Tf 72 720 Td (Zlib framed hello) Tj ET\n"). */
|
|
static const unsigned char ZLIB[] = {
|
|
0x78, 0xda, 0x73, 0x0a, 0x51, 0xd0, 0x77, 0x33, 0x54, 0x30, 0x34, 0x52,
|
|
0x08, 0x49, 0x53, 0x30, 0x37, 0x02, 0x22, 0x03, 0x85, 0x90, 0x14, 0x05,
|
|
0x8d, 0xa8, 0x9c, 0xcc, 0x24, 0x85, 0xb4, 0xa2, 0xc4, 0xdc, 0xd4, 0x14,
|
|
0x85, 0x8c, 0xd4, 0x9c, 0x9c, 0x7c, 0x4d, 0x85, 0x90, 0x2c, 0x05, 0xd7,
|
|
0x10, 0x2e, 0x00, 0x33, 0x0f, 0x0d, 0x3a};
|
|
|
|
static void test_zlib_flate(void) {
|
|
char pdf[2048];
|
|
int n = snprintf(pdf, sizeof(pdf),
|
|
"%%PDF-1.4\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font "
|
|
"<< /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
|
"4 0 obj\n<< /Type /Font /Subtype /Type1 >>\nendobj\n"
|
|
"5 0 obj\n<< /Filter /FlateDecode >>\nstream\n");
|
|
memcpy(pdf + n, ZLIB, sizeof(ZLIB));
|
|
n += (int)sizeof(ZLIB);
|
|
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
|
"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(has(t, "Zlib framed hello"), "zlib header skipped, text inflated");
|
|
free(t);
|
|
}
|
|
|
|
/* 2c. an inline image: its bytes are not text and must be skipped. */
|
|
|
|
static const char INLINE_IMG[] =
|
|
"%PDF-1.4\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
|
">> >> /Contents 5 0 R >>\nendobj\n"
|
|
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n"
|
|
"5 0 obj\n<< >>\nstream\n"
|
|
"BT /F1 12 Tf 72 720 Td (Before the image) Tj ET\n"
|
|
"q 20 0 0 20 72 700 cm\n"
|
|
"BI /W 8 /H 8 /CS /RGB /BPC 8 /F /AHx ID\n"
|
|
"00ff0000ff00ff00ff00ff00 8123<0001> EI Q\n"
|
|
"BT /F1 12 Tf 72 680 Td (After the image) Tj ET\n"
|
|
"endstream\nendobj\n"
|
|
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
|
|
|
static void test_inline_image(void) {
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = extract(INLINE_IMG, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(has(t, "Before the image"), "text before the inline image");
|
|
CHECK(has(t, "After the image"), "text after it is still found");
|
|
CHECK(!has(t, "00ff0000"), "the image bytes were skipped");
|
|
CHECK(!has(t, "8123"), "its hex data was skipped too");
|
|
free(t);
|
|
}
|
|
|
|
/* --- 3. CID font with a ToUnicode CMap ------------------------------ */
|
|
|
|
static const char CID[] =
|
|
"%PDF-1.5\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
|
">> >> /Contents 5 0 R >>\nendobj\n"
|
|
"4 0 obj\n<< /Type /Font /Subtype /Type0 /Encoding /Identity-H "
|
|
"/ToUnicode 6 0 R >>\nendobj\n"
|
|
"5 0 obj\n<< >>\nstream\n"
|
|
"BT /F1 12 Tf 72 720 Td <0001000200030004> Tj ET\n"
|
|
"endstream\nendobj\n"
|
|
"6 0 obj\n<< >>\nstream\n"
|
|
"1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n"
|
|
"2 beginbfchar\n<0001> <4F60>\n<0002> <597D>\nendbfchar\n"
|
|
"1 beginbfrange\n<0003> <0004> <0041>\nendbfrange\n"
|
|
"endcmap\nendstream\nendobj\n"
|
|
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
|
|
|
static void test_cid(void) {
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = extract(CID, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(has(t, "\xe4\xbd\xa0\xe5\xa5\xbd"), "bfchar maps to 你好");
|
|
CHECK(has(t, "AB"), "bfrange walks the code points");
|
|
free(t);
|
|
}
|
|
|
|
/* --- 4. objects inside an object stream ----------------------------- */
|
|
|
|
static void test_objstm(void) {
|
|
char pdf[4096];
|
|
const char *page = "<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 "
|
|
"4 0 R >> >> /Contents 5 0 R >>";
|
|
const char *font = "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>";
|
|
/* the stream is "num1 off1 num2 off2" -- /N and /First are in the dict --
|
|
* and offsets count from /First, which is where the bodies start, so
|
|
* /First is the length of the pairs list itself */
|
|
char hdr[64];
|
|
int off4 = (int)strlen(page) + 1; /* object 4 follows the page */
|
|
int first = snprintf(hdr, sizeof(hdr), "3 0 4 %d ", off4);
|
|
int n = snprintf(pdf, sizeof(pdf),
|
|
"%%PDF-1.5\n"
|
|
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
|
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
|
"5 0 obj\n<< >>\nstream\n"
|
|
"BT /F1 12 Tf 72 720 Td (Inside an object stream) Tj ET\n"
|
|
"endstream\nendobj\n"
|
|
"6 0 obj\n<< /Type /ObjStm /N 2 /First %d >>\nstream\n"
|
|
"%s%s\n%s",
|
|
first, hdr, page, font);
|
|
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
|
"\nendstream\nendobj\n"
|
|
"trailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
|
char err[256] = "";
|
|
int pages = 0;
|
|
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
|
CHECK(t != NULL, err);
|
|
CHECK(pages == 1, "one page through the object stream");
|
|
CHECK(has(t, "Inside an object stream"), "text of the page in the ObjStm");
|
|
free(t);
|
|
}
|
|
|
|
/* --- 5. error paths -------------------------------------------------- */
|
|
|
|
static void test_errors(void) {
|
|
char err[256] = "";
|
|
char *t = extract("%PDF-1.7\ntrailer\n<< /Root 1 0 R /Encrypt 9 0 R >>\n",
|
|
NULL, err, sizeof(err));
|
|
CHECK(t == NULL, "encrypted file is refused");
|
|
CHECK(has(err, "encrypted"), "says it is encrypted");
|
|
|
|
err[0] = 0;
|
|
t = extract("hello, not a pdf at all\n", NULL, err, sizeof(err));
|
|
CHECK(t == NULL, "non-PDF is refused");
|
|
CHECK(has(err, "not a PDF"), "says it is not a PDF");
|
|
|
|
err[0] = 0;
|
|
t = extract("%PDF-1.4\n1 0 obj\n<< >>\nendobj\n", NULL, err, sizeof(err));
|
|
CHECK(t == NULL, "a PDF without text is refused");
|
|
CHECK(has(err, "no extractable text") || has(err, "no readable pages"),
|
|
"explains the missing text");
|
|
}
|
|
|
|
int main(void) {
|
|
test_plain();
|
|
test_flate();
|
|
test_zlib_flate();
|
|
test_inline_image();
|
|
test_cid();
|
|
test_objstm();
|
|
test_errors();
|
|
if (failures) {
|
|
fprintf(stderr, "test_pdf_text: %d failure(s)\n", failures);
|
|
return 1;
|
|
}
|
|
printf("test_pdf_text: OK\n");
|
|
return 0;
|
|
}
|