v0.8.6: general mode, clipboard text, PDF text extraction
Working mode: --mode coding|general (CST_MODE, settings.json, /mode, WebUI run settings). general swaps the default framing for a general assistant one -- workflow, safety notes, tool wording (run_shell, the subagents) -- and /status shows it; minimal (the paper profile) stays byte-frozen. The prompt state carries the mode so a switch leaves a system note in history. Clipboard: clipboard_read / clipboard_write tools (16 MiB cap read, default 64 KiB window; writing is a WRITE permission with a preview) on pbcopy / wl-copy / xclip / pbpaste / wl-paste / xclip and the Win32 clipboard. In the TUI, /attach paste falls back to clipboard text when there is no image on it. PDF: a dependency-free text extractor (src/pdf_text.c): inflate, Flate/ASCIIHex/ASCII85, PNG/TIFF predictors, layout-independent object scan, object streams, page tree walk with inherited resources, content lexer (Tj/TJ/'/" and the Td/TD/T*/Tm spacing heuristics), per-font ToUnicode CMaps and inline images skipped. Encrypted files and pages with no text layer are refused with the reason. Attachments: /attach (a path or files dropped on the terminal) now take images, PDFs and text. A PDF is stored as its text layer with the mime still saying application/pdf and its page count; anything else has to be UTF-8 text. Documents travel inline as text parts, so CST_NO_IMAGES and text-only models no longer drop them, while images keep their old path. Tests: test_pdf_text (plain, bare and zlib-framed Flate, inline image, CID + ToUnicode, object stream, error paths), documents in test_attach, inlined documents in test_llm_body, mode in test_config.
This commit is contained in:
WHC
@@ -325,6 +325,100 @@ static void test_import_file(const char *root) {
|
||||
"missing file refused");
|
||||
}
|
||||
|
||||
/* Documents: text is stored as text, a PDF as its text layer while the mime
|
||||
* still says PDF, and neither is an image as far as the wire is concerned. */
|
||||
static void test_documents(const char *root) {
|
||||
char session[1024], src[1024];
|
||||
snprintf(session, sizeof(session), "%.900s/sessions/doc.jsonl", root);
|
||||
char err[256] = "";
|
||||
|
||||
const char *text = "# Notes\n\nFive painters, one canvas.\n";
|
||||
snprintf(src, sizeof(src), "%.900s/notes.md", root);
|
||||
FILE *f = fopen(src, "wb");
|
||||
EXPECT(f != NULL, "write the text file");
|
||||
if (f) {
|
||||
fwrite(text, 1, strlen(text), f);
|
||||
fclose(f);
|
||||
}
|
||||
cst_attachment_t *a = cst_attach_import_file(session, src, err, sizeof(err));
|
||||
EXPECT(a != NULL, "text file imported");
|
||||
if (a) {
|
||||
EXPECT(strcmp(a->mime, "text/plain") == 0, "text mime");
|
||||
EXPECT(!cst_attach_is_image(a) && cst_attach_is_text(a), "not an image");
|
||||
EXPECT(a->pages == 0, "no pages for text");
|
||||
EXPECT(strstr(a->rel, ".txt") != NULL, "text stored as .txt");
|
||||
EXPECT(a->bytes == (long long)strlen(text), "text size");
|
||||
size_t n = 0;
|
||||
char *back = cst_attach_read(session, a, &n, err, sizeof(err));
|
||||
EXPECT(back && n == strlen(text) && memcmp(back, text, n) == 0,
|
||||
"text reads back exactly");
|
||||
free(back);
|
||||
EXPECT(cst_attach_data_url(session, a) == NULL,
|
||||
"a document has no data url");
|
||||
char desc[128];
|
||||
cst_attach_describe(a, desc, sizeof(desc));
|
||||
EXPECT(strstr(desc, "text") != NULL, "describe says text");
|
||||
cst_attach_free(a);
|
||||
}
|
||||
|
||||
/* a one-page PDF with a text layer */
|
||||
static const char PDF[] =
|
||||
"%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
||||
">> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\n"
|
||||
"endobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT /F1 12 Tf 72 720 Td (Attached paper abstract) Tj ET\n"
|
||||
"endstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
||||
snprintf(src, sizeof(src), "%.900s/paper.pdf", root);
|
||||
f = fopen(src, "wb");
|
||||
EXPECT(f != NULL, "write the PDF");
|
||||
if (f) {
|
||||
fwrite(PDF, 1, strlen(PDF), f);
|
||||
fclose(f);
|
||||
}
|
||||
err[0] = 0;
|
||||
a = cst_attach_import_file(session, src, err, sizeof(err));
|
||||
EXPECT(a != NULL, "PDF imported");
|
||||
if (a) {
|
||||
EXPECT(strcmp(a->mime, "application/pdf") == 0, "the PDF keeps its mime");
|
||||
EXPECT(a->pages == 1, "page count comes from the extractor");
|
||||
EXPECT(strstr(a->rel, ".txt") != NULL, "the text layer is what is stored");
|
||||
EXPECT(cst_attach_is_text(a), "documents are text on the wire");
|
||||
char desc[128];
|
||||
cst_attach_describe(a, desc, sizeof(desc));
|
||||
EXPECT(strstr(desc, "pdf") != NULL && strstr(desc, "1 page") != NULL,
|
||||
"describe names the PDF and its pages");
|
||||
size_t n = 0;
|
||||
char *back = cst_attach_read(session, a, &n, err, sizeof(err));
|
||||
EXPECT(back && strstr(back, "Attached paper abstract") != NULL,
|
||||
"the extracted text reads back");
|
||||
free(back);
|
||||
cst_attach_free(a);
|
||||
}
|
||||
err[0] = 0;
|
||||
EXPECT(cst_attach_import_file(session, "/nonexistent/x.pdf", err,
|
||||
sizeof(err)) == NULL,
|
||||
"a missing PDF is refused");
|
||||
|
||||
/* neither image, PDF nor text: a binary blob */
|
||||
static const char binary[] = "\x01\x02\x03\x00\x04\x05";
|
||||
snprintf(src, sizeof(src), "%.900s/blob.bin", root);
|
||||
f = fopen(src, "wb");
|
||||
if (f) {
|
||||
fwrite(binary, 1, sizeof(binary), f);
|
||||
fclose(f);
|
||||
}
|
||||
err[0] = 0;
|
||||
EXPECT(cst_attach_import_file(session, src, err, sizeof(err)) == NULL,
|
||||
"a binary file is refused");
|
||||
EXPECT(strstr(err, "not an image") != NULL, "err says what was expected");
|
||||
}
|
||||
|
||||
static void test_json_roundtrip(void) {
|
||||
cst_attachment_t list[3];
|
||||
memset(list, 0, sizeof(list));
|
||||
@@ -344,6 +438,8 @@ static void test_json_roundtrip(void) {
|
||||
list[1].bytes = 4096;
|
||||
snprintf(list[1].detail, sizeof(list[1].detail), "high");
|
||||
snprintf(list[1].origin, sizeof(list[1].origin), "view_image");
|
||||
snprintf(list[1].mime, sizeof(list[1].mime), "application/pdf");
|
||||
list[1].pages = 12;
|
||||
|
||||
snprintf(list[2].id, sizeof(list[2].id), "img-3");
|
||||
snprintf(list[2].name, sizeof(list[2].name), "c.webp");
|
||||
@@ -365,6 +461,9 @@ static void test_json_roundtrip(void) {
|
||||
EXPECT(strcmp(back[1].detail, "high") == 0 &&
|
||||
strcmp(back[1].origin, "view_image") == 0,
|
||||
"detail and origin survive");
|
||||
EXPECT(strcmp(back[1].mime, "application/pdf") == 0 && back[1].pages == 12,
|
||||
"a document keeps its mime and page count");
|
||||
EXPECT(back[2].pages == 0, "pages default to 0");
|
||||
EXPECT(strcmp(back[2].mime, "image/webp") == 0 && back[2].width == 0,
|
||||
"entry without dimensions");
|
||||
EXPECT(strcmp(back[0].detail, "auto") == 0, "missing detail defaults");
|
||||
@@ -530,6 +629,7 @@ int main(void) {
|
||||
test_dib();
|
||||
test_store(root);
|
||||
test_import_file(root);
|
||||
test_documents(root);
|
||||
test_json_roundtrip();
|
||||
test_lists();
|
||||
test_agent_pending(root);
|
||||
|
||||
@@ -106,11 +106,13 @@ static void test_settings(void) {
|
||||
snprintf(cfg.provider, sizeof(cfg.provider), "openai");
|
||||
snprintf(cfg.model, sizeof(cfg.model), "gpt-5");
|
||||
snprintf(cfg.effort, sizeof(cfg.effort), "max");
|
||||
snprintf(cfg.mode, sizeof(cfg.mode), "general");
|
||||
CHECK(cst_settings_remember(&cfg) == 0);
|
||||
|
||||
CHECK(cst_settings_load(&s) == 0);
|
||||
CHECK(strcmp(s.provider, "openai") == 0);
|
||||
CHECK(strcmp(s.effort, "max") == 0);
|
||||
CHECK(strcmp(s.mode, "general") == 0);
|
||||
CHECK(strcmp(cst_settings_model(&s, "deepseek"), "deepseek-v4-pro") == 0);
|
||||
CHECK(strcmp(cst_settings_model(&s, "openai"), "gpt-5") == 0);
|
||||
CHECK(s.nmodels == 2);
|
||||
@@ -145,6 +147,24 @@ static void test_effort(void) {
|
||||
free(body);
|
||||
}
|
||||
|
||||
static void test_mode(void) {
|
||||
CHECK(cst_mode_valid("coding") && cst_mode_valid("general"));
|
||||
CHECK(!cst_mode_valid("code") && !cst_mode_valid(NULL));
|
||||
CHECK(!cst_mode_valid(""));
|
||||
CHECK(strcmp(cst_mode_name("general"), "general") == 0);
|
||||
CHECK(strcmp(cst_mode_name(NULL), CST_DEFAULT_MODE) == 0);
|
||||
CHECK(strcmp(cst_mode_name("nonsense"), "coding") == 0);
|
||||
|
||||
CHECK(cst_settings_remember_mode("general") == 0);
|
||||
cst_settings_t s;
|
||||
CHECK(cst_settings_load(&s) == 0);
|
||||
CHECK(strcmp(s.mode, "general") == 0);
|
||||
/* an unknown value is recorded as the default, not as itself */
|
||||
CHECK(cst_settings_remember_mode("junk") == 0);
|
||||
CHECK(cst_settings_load(&s) == 0);
|
||||
CHECK(strcmp(s.mode, "coding") == 0);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
CHECK(test_tmpdir(g_dir, sizeof(g_dir), "cst-config") == 0);
|
||||
setenv("XDG_CONFIG_HOME", g_dir, 1);
|
||||
@@ -158,6 +178,7 @@ int main(void) {
|
||||
test_creds();
|
||||
test_settings();
|
||||
test_effort();
|
||||
test_mode();
|
||||
|
||||
CHECK(test_rmrf(g_dir) == 0);
|
||||
printf("test_config: OK\n");
|
||||
|
||||
@@ -406,9 +406,118 @@ static void test_images_on(void) {
|
||||
cst_attach_free(img);
|
||||
}
|
||||
|
||||
/* Every text part of message i, joined (malloc'd, "" when there is none). */
|
||||
static char *text_of(const cJSON *root, int i) {
|
||||
const cJSON *m =
|
||||
cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "messages"), i);
|
||||
const cJSON *c = cJSON_GetObjectItemCaseSensitive(m, "content");
|
||||
if (cJSON_IsString(c)) {
|
||||
size_t n = strlen(c->valuestring);
|
||||
char *out = malloc(n + 1);
|
||||
if (out)
|
||||
memcpy(out, c->valuestring, n + 1);
|
||||
return out;
|
||||
}
|
||||
size_t len = 1;
|
||||
const cJSON *p, *t, *s;
|
||||
cJSON_ArrayForEach(p, c) {
|
||||
t = cJSON_GetObjectItemCaseSensitive(p, "type");
|
||||
s = cJSON_GetObjectItemCaseSensitive(p, "text");
|
||||
if (cJSON_IsString(t) && strcmp(t->valuestring, "text") == 0 &&
|
||||
cJSON_IsString(s))
|
||||
len += strlen(s->valuestring) + 1;
|
||||
}
|
||||
char *out = calloc(1, len);
|
||||
if (!out)
|
||||
return NULL;
|
||||
cJSON_ArrayForEach(p, c) {
|
||||
t = cJSON_GetObjectItemCaseSensitive(p, "type");
|
||||
s = cJSON_GetObjectItemCaseSensitive(p, "text");
|
||||
if (cJSON_IsString(t) && strcmp(t->valuestring, "text") == 0 &&
|
||||
cJSON_IsString(s)) {
|
||||
strcat(out, s->valuestring);
|
||||
strcat(out, "\n");
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* A document is inlined as text, so a text-only model and a message that
|
||||
* also carries an image both keep it; the image is what becomes a note. */
|
||||
static void test_document_parts(void) {
|
||||
char dir[1024], sess[1100], err[256], src[1200];
|
||||
if (test_tmpdir(dir, sizeof(dir), "cst-body-doc") != 0) {
|
||||
EXPECT(0, "scratch dir");
|
||||
return;
|
||||
}
|
||||
snprintf(sess, sizeof(sess), "%s/s.jsonl", dir);
|
||||
snprintf(src, sizeof(src), "%s/paper.pdf", dir);
|
||||
static const char PDF[] =
|
||||
"%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
||||
">> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\n"
|
||||
"endobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT /F1 12 Tf 72 720 Td (Inlined paper text) Tj ET\n"
|
||||
"endstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
||||
FILE *f = fopen(src, "wb");
|
||||
EXPECT(f != NULL, "write the PDF");
|
||||
if (!f)
|
||||
return;
|
||||
fwrite(PDF, 1, strlen(PDF), f);
|
||||
fclose(f);
|
||||
cst_attachment_t *doc = cst_attach_import_file(sess, src, err, sizeof(err));
|
||||
EXPECT(doc != NULL, "PDF stored");
|
||||
if (!doc)
|
||||
return;
|
||||
static const unsigned char png[32] = {0x89, 'P', 'N', 'G', '\r', '\n', 0x1A,
|
||||
'\n', 0, 0, 0, 13, 'I', 'H', 'D', 'R',
|
||||
0, 0, 0, 2, 0, 0, 0, 2};
|
||||
cst_attachment_t *img = cst_attach_import_bytes(
|
||||
sess, "shot.png", (const char *)png, sizeof(png), err, sizeof(err));
|
||||
EXPECT(img != NULL, "image stored");
|
||||
if (!img) {
|
||||
cst_attach_free(doc);
|
||||
return;
|
||||
}
|
||||
cst_attachment_t both[2] = {*doc, *img};
|
||||
cst_message_t m[2];
|
||||
m[0] = msg(CST_ROLE_USER, "what does this say?");
|
||||
m[0].attach = both;
|
||||
m[0].nattach = 2;
|
||||
m[1] = msg(CST_ROLE_USER, "thanks");
|
||||
cst_llm_request_t r;
|
||||
memset(&r, 0, sizeof(r));
|
||||
r.model = "deepseek-flash";
|
||||
r.msgs = m;
|
||||
r.nmsgs = 2;
|
||||
r.images_off = 1; /* a text-only model */
|
||||
r.session_path = sess;
|
||||
cJSON *j = build_req(&r);
|
||||
char *txt = text_of(j, 0);
|
||||
EXPECT(txt && strstr(txt, "[document \"paper.pdf\"") &&
|
||||
strstr(txt, "Inlined paper text") &&
|
||||
strstr(txt, "[-- end of document --]"),
|
||||
"the PDF text is inlined even when images are off");
|
||||
EXPECT(txt && strstr(txt, "[image \"shot.png\"") && strstr(txt, "not sent"),
|
||||
"the image beside it is still only a note");
|
||||
free(txt);
|
||||
char *s = cJSON_PrintUnformatted(j);
|
||||
EXPECT(s && !strstr(s, "image_url"), "no image part with images off");
|
||||
free(s);
|
||||
cJSON_Delete(j);
|
||||
cst_attach_free(doc);
|
||||
cst_attach_free(img);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
test_images_off();
|
||||
test_images_on();
|
||||
test_document_parts();
|
||||
test_reasoning_and_prefix();
|
||||
test_cache_hints();
|
||||
test_tools_snapshot();
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
/* Text extraction from PDFs: hand-made fixtures for the shapes that matter
|
||||
* (plain content, FlateDecode, CID fonts with ToUnicode CMaps, object
|
||||
* streams) plus the two error paths (encrypted, not a PDF). */
|
||||
|
||||
#include "cst_pdf.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static int failures = 0;
|
||||
|
||||
#define CHECK(cond, what) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, what); \
|
||||
failures++; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static char *extract(const char *pdf, int *pages, char *err, size_t err_sz) {
|
||||
return cst_pdf_text(pdf, strlen(pdf), pages, err, err_sz);
|
||||
}
|
||||
|
||||
static int has(const char *hay, const char *needle) {
|
||||
return hay && strstr(hay, needle) != NULL;
|
||||
}
|
||||
|
||||
/* --- 1. plain, uncompressed content, two lines via Td --------------- */
|
||||
|
||||
static const char PLAIN[] =
|
||||
"%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
|
||||
"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT\n/F1 12 Tf\n72 720 Td\n(Line one) Tj\n0 -14 Td\n"
|
||||
"(A \\(paren\\) and a \\\\ backslash) Tj\nET\n"
|
||||
"endstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
||||
|
||||
static void test_plain(void) {
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = extract(PLAIN, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(pages == 1, "one page");
|
||||
CHECK(has(t, "Line one"), "first line");
|
||||
CHECK(has(t, "A (paren) and a \\ backslash"), "escapes");
|
||||
CHECK(!has(t, "\r"), "no CR left");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* --- 2. FlateDecode content stream ---------------------------------- */
|
||||
|
||||
/* raw deflate (zlib, level 9, no header) of
|
||||
* "BT /F1 12 Tf 72 720 Td (Hello, compressed world) Tj ET\n" */
|
||||
static const unsigned char FLATE[] = {
|
||||
0x73, 0x0a, 0x51, 0xd0, 0x77, 0x33, 0x54, 0x30, 0x34, 0x52, 0x08, 0x49,
|
||||
0x53, 0x30, 0x37, 0x02, 0x22, 0x03, 0x85, 0x90, 0x14, 0x05, 0x0d, 0x8f,
|
||||
0xd4, 0x9c, 0x9c, 0x7c, 0x1d, 0x85, 0xe4, 0xfc, 0xdc, 0x82, 0xa2, 0xd4,
|
||||
0xe2, 0xe2, 0xd4, 0x14, 0x85, 0xf2, 0xfc, 0xa2, 0x9c, 0x14, 0x4d, 0x85,
|
||||
0x90, 0x2c, 0x05, 0xd7, 0x10, 0x2e, 0x00};
|
||||
|
||||
static void test_flate(void) {
|
||||
char pdf[2048];
|
||||
int n = snprintf(pdf, sizeof(pdf),
|
||||
"%%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font "
|
||||
"<< /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 >>\nendobj\n"
|
||||
"5 0 obj\n<< /Filter /FlateDecode >>\nstream\n");
|
||||
memcpy(pdf + n, FLATE, sizeof(FLATE));
|
||||
n += (int)sizeof(FLATE);
|
||||
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
||||
"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(has(t, "Hello, compressed world"), "inflated text");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* 2b. zlib-framed FlateDecode (RFC 1950), which is what real writers emit:
|
||||
* zlib.compress(b"BT /F1 12 Tf 72 720 Td (Zlib framed hello) Tj ET\n"). */
|
||||
static const unsigned char ZLIB[] = {
|
||||
0x78, 0xda, 0x73, 0x0a, 0x51, 0xd0, 0x77, 0x33, 0x54, 0x30, 0x34, 0x52,
|
||||
0x08, 0x49, 0x53, 0x30, 0x37, 0x02, 0x22, 0x03, 0x85, 0x90, 0x14, 0x05,
|
||||
0x8d, 0xa8, 0x9c, 0xcc, 0x24, 0x85, 0xb4, 0xa2, 0xc4, 0xdc, 0xd4, 0x14,
|
||||
0x85, 0x8c, 0xd4, 0x9c, 0x9c, 0x7c, 0x4d, 0x85, 0x90, 0x2c, 0x05, 0xd7,
|
||||
0x10, 0x2e, 0x00, 0x33, 0x0f, 0x0d, 0x3a};
|
||||
|
||||
static void test_zlib_flate(void) {
|
||||
char pdf[2048];
|
||||
int n = snprintf(pdf, sizeof(pdf),
|
||||
"%%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font "
|
||||
"<< /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 >>\nendobj\n"
|
||||
"5 0 obj\n<< /Filter /FlateDecode >>\nstream\n");
|
||||
memcpy(pdf + n, ZLIB, sizeof(ZLIB));
|
||||
n += (int)sizeof(ZLIB);
|
||||
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
||||
"\nendstream\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(has(t, "Zlib framed hello"), "zlib header skipped, text inflated");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* 2c. an inline image: its bytes are not text and must be skipped. */
|
||||
|
||||
static const char INLINE_IMG[] =
|
||||
"%PDF-1.4\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
||||
">> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT /F1 12 Tf 72 720 Td (Before the image) Tj ET\n"
|
||||
"q 20 0 0 20 72 700 cm\n"
|
||||
"BI /W 8 /H 8 /CS /RGB /BPC 8 /F /AHx ID\n"
|
||||
"00ff0000ff00ff00ff00ff00 8123<0001> EI Q\n"
|
||||
"BT /F1 12 Tf 72 680 Td (After the image) Tj ET\n"
|
||||
"endstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
||||
|
||||
static void test_inline_image(void) {
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = extract(INLINE_IMG, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(has(t, "Before the image"), "text before the inline image");
|
||||
CHECK(has(t, "After the image"), "text after it is still found");
|
||||
CHECK(!has(t, "00ff0000"), "the image bytes were skipped");
|
||||
CHECK(!has(t, "8123"), "its hex data was skipped too");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* --- 3. CID font with a ToUnicode CMap ------------------------------ */
|
||||
|
||||
static const char CID[] =
|
||||
"%PDF-1.5\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R "
|
||||
">> >> /Contents 5 0 R >>\nendobj\n"
|
||||
"4 0 obj\n<< /Type /Font /Subtype /Type0 /Encoding /Identity-H "
|
||||
"/ToUnicode 6 0 R >>\nendobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT /F1 12 Tf 72 720 Td <0001000200030004> Tj ET\n"
|
||||
"endstream\nendobj\n"
|
||||
"6 0 obj\n<< >>\nstream\n"
|
||||
"1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n"
|
||||
"2 beginbfchar\n<0001> <4F60>\n<0002> <597D>\nendbfchar\n"
|
||||
"1 beginbfrange\n<0003> <0004> <0041>\nendbfrange\n"
|
||||
"endcmap\nendstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n";
|
||||
|
||||
static void test_cid(void) {
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = extract(CID, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(has(t, "\xe4\xbd\xa0\xe5\xa5\xbd"), "bfchar maps to 你好");
|
||||
CHECK(has(t, "AB"), "bfrange walks the code points");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* --- 4. objects inside an object stream ----------------------------- */
|
||||
|
||||
static void test_objstm(void) {
|
||||
char pdf[4096];
|
||||
const char *page = "<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 "
|
||||
"4 0 R >> >> /Contents 5 0 R >>";
|
||||
const char *font = "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>";
|
||||
/* the stream is "num1 off1 num2 off2" -- /N and /First are in the dict --
|
||||
* and offsets count from /First, which is where the bodies start, so
|
||||
* /First is the length of the pairs list itself */
|
||||
char hdr[64];
|
||||
int off4 = (int)strlen(page) + 1; /* object 4 follows the page */
|
||||
int first = snprintf(hdr, sizeof(hdr), "3 0 4 %d ", off4);
|
||||
int n = snprintf(pdf, sizeof(pdf),
|
||||
"%%PDF-1.5\n"
|
||||
"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
|
||||
"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
|
||||
"5 0 obj\n<< >>\nstream\n"
|
||||
"BT /F1 12 Tf 72 720 Td (Inside an object stream) Tj ET\n"
|
||||
"endstream\nendobj\n"
|
||||
"6 0 obj\n<< /Type /ObjStm /N 2 /First %d >>\nstream\n"
|
||||
"%s%s\n%s",
|
||||
first, hdr, page, font);
|
||||
n += snprintf(pdf + n, sizeof(pdf) - (size_t)n,
|
||||
"\nendstream\nendobj\n"
|
||||
"trailer\n<< /Root 1 0 R >>\n%%EOF\n");
|
||||
char err[256] = "";
|
||||
int pages = 0;
|
||||
char *t = cst_pdf_text(pdf, (size_t)n, &pages, err, sizeof(err));
|
||||
CHECK(t != NULL, err);
|
||||
CHECK(pages == 1, "one page through the object stream");
|
||||
CHECK(has(t, "Inside an object stream"), "text of the page in the ObjStm");
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* --- 5. error paths -------------------------------------------------- */
|
||||
|
||||
static void test_errors(void) {
|
||||
char err[256] = "";
|
||||
char *t = extract("%PDF-1.7\ntrailer\n<< /Root 1 0 R /Encrypt 9 0 R >>\n",
|
||||
NULL, err, sizeof(err));
|
||||
CHECK(t == NULL, "encrypted file is refused");
|
||||
CHECK(has(err, "encrypted"), "says it is encrypted");
|
||||
|
||||
err[0] = 0;
|
||||
t = extract("hello, not a pdf at all\n", NULL, err, sizeof(err));
|
||||
CHECK(t == NULL, "non-PDF is refused");
|
||||
CHECK(has(err, "not a PDF"), "says it is not a PDF");
|
||||
|
||||
err[0] = 0;
|
||||
t = extract("%PDF-1.4\n1 0 obj\n<< >>\nendobj\n", NULL, err, sizeof(err));
|
||||
CHECK(t == NULL, "a PDF without text is refused");
|
||||
CHECK(has(err, "no extractable text") || has(err, "no readable pages"),
|
||||
"explains the missing text");
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
test_plain();
|
||||
test_flate();
|
||||
test_zlib_flate();
|
||||
test_inline_image();
|
||||
test_cid();
|
||||
test_objstm();
|
||||
test_errors();
|
||||
if (failures) {
|
||||
fprintf(stderr, "test_pdf_text: %d failure(s)\n", failures);
|
||||
return 1;
|
||||
}
|
||||
printf("test_pdf_text: OK\n");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user