From b3c8baf856e3797d3ba5d14dad999ff50b2a9a9f Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:17:51 +0200 Subject: [PATCH] feat(rtf): read an rtf as a text document Stage 1 of `internal/rtf/PLAN.md`: an rtf opens and renders instead of reaching `open_strategy`'s fallthrough and throwing `UnknownFileType`. A pull-based `Tokenizer` over the rtf byte grammar, a `State` group stack, and a parser building `root -> paragraph -> (text | line break)` so the generic html renderer and every binding get it for free. Text, its encoding (`\ansi`, `\mac`, `\ansicpgN`, `\'hh`, `\uN` with signed folding and surrogate pairs, `\ucN` skipping), paragraphs, line breaks and tabs. Leniency is the spec here: unknown control words and unimplemented `{\*` destinations are ignored, an unmatched `}` is dropped. A group left open at EOF, a bad hex digit, a `\binN` past EOF and excessive nesting throw. Character and paragraph formatting, page layout, tables and pictures are stages 2-5 and are not read yet. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01N47M1zVGirwkpbtXXctCAb --- AGENTS.md | 1 + CHANGELOG.md | 6 + CMakeLists.txt | 7 + src/odr/exceptions.cpp | 2 + src/odr/exceptions.hpp | 5 + src/odr/internal/file_type_table.cpp | 10 +- src/odr/internal/open_strategy.cpp | 18 + src/odr/internal/rtf/AGENTS.md | 97 +++ src/odr/internal/rtf/PLAN.md | 17 +- src/odr/internal/rtf/rtf_document.cpp | 189 ++++++ src/odr/internal/rtf/rtf_document.hpp | 30 + src/odr/internal/rtf/rtf_element_registry.cpp | 94 +++ src/odr/internal/rtf/rtf_element_registry.hpp | 52 ++ src/odr/internal/rtf/rtf_file.cpp | 47 ++ src/odr/internal/rtf/rtf_file.hpp | 33 + src/odr/internal/rtf/rtf_parser.cpp | 566 ++++++++++++++++++ src/odr/internal/rtf/rtf_parser.hpp | 16 + src/odr/internal/rtf/rtf_state.cpp | 28 + src/odr/internal/rtf/rtf_state.hpp | 41 ++ src/odr/internal/rtf/rtf_token.hpp | 42 ++ src/odr/internal/rtf/rtf_tokenizer.cpp | 191 ++++++ src/odr/internal/rtf/rtf_tokenizer.hpp | 46 ++ test/CMakeLists.txt | 3 + test/src/html_output_test.cpp | 2 +- test/src/internal/rtf/rtf_document_test.cpp | 191 ++++++ test/src/internal/rtf/rtf_tokenizer_test.cpp | 138 +++++ 26 files changed, 1861 insertions(+), 11 deletions(-) create mode 100644 src/odr/internal/rtf/AGENTS.md create mode 100644 src/odr/internal/rtf/rtf_document.cpp create mode 100644 src/odr/internal/rtf/rtf_document.hpp create mode 100644 src/odr/internal/rtf/rtf_element_registry.cpp create mode 100644 src/odr/internal/rtf/rtf_element_registry.hpp create mode 100644 src/odr/internal/rtf/rtf_file.cpp create mode 100644 src/odr/internal/rtf/rtf_file.hpp create mode 100644 src/odr/internal/rtf/rtf_parser.cpp create mode 100644 src/odr/internal/rtf/rtf_parser.hpp create mode 100644 src/odr/internal/rtf/rtf_state.cpp create mode 100644 src/odr/internal/rtf/rtf_state.hpp create mode 100644 src/odr/internal/rtf/rtf_token.hpp create mode 100644 src/odr/internal/rtf/rtf_tokenizer.cpp create mode 100644 src/odr/internal/rtf/rtf_tokenizer.hpp create mode 100644 test/src/internal/rtf/rtf_document_test.cpp create mode 100644 test/src/internal/rtf/rtf_tokenizer_test.cpp diff --git a/AGENTS.md b/AGENTS.md index d7b27100..3476ef19 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | | `src/odr/internal/pdf/` | PDF (own parser). | +| `src/odr/internal/rtf/` | RTF, read as a text document; see [`rtf/AGENTS.md`](src/odr/internal/rtf/AGENTS.md) + [`rtf/PLAN.md`](src/odr/internal/rtf/PLAN.md). | | `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | | `src/odr/internal/{csv,json,text,svm}/` | Smaller formats. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc85b98..73149d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- An rtf opens and renders instead of throwing `UnknownFileType`. Text, its + encoding (`\ansicpgN`, `\'hh`, `\uN` including emoji), paragraphs, line + breaks and tabs; character and paragraph formatting, tables and pictures are + not read yet. `FileType::rich_text_format` now reports `DocumentType::text` + and the `open` / `translate_html` / `color_scheme` capabilities. + ## v6.10.1 - 2026-08-21 - A linked image in a docx or xlsx (`embed_images = false`) is named relative diff --git a/CMakeLists.txt b/CMakeLists.txt index 04feb219..4fe02fd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,6 +239,13 @@ set(ODR_SOURCE_FILES "src/odr/internal/font/sfnt_transform.cpp" "src/odr/internal/font/font_file.cpp" + "src/odr/internal/rtf/rtf_document.cpp" + "src/odr/internal/rtf/rtf_element_registry.cpp" + "src/odr/internal/rtf/rtf_file.cpp" + "src/odr/internal/rtf/rtf_parser.cpp" + "src/odr/internal/rtf/rtf_state.cpp" + "src/odr/internal/rtf/rtf_tokenizer.cpp" + "src/odr/internal/svg/svg_file.cpp" "src/odr/internal/svm/svm_file.cpp" diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index aaa8d361..50c9008f 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -72,6 +72,8 @@ NoXmlFile::NoXmlFile() : Exception("not an xml file") {} NoSvgFile::NoSvgFile() : Exception("not an svg file") {} +NoRtfFile::NoRtfFile() : Exception("not an rtf file") {} + UnsupportedCryptoAlgorithm::UnsupportedCryptoAlgorithm() : Exception("unsupported crypto algorithm") {} diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index c9d55fc1..662d177a 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -152,6 +152,11 @@ struct NoSvgFile final : Exception { NoSvgFile(); }; +/// @brief No RTF file exception +struct NoRtfFile final : Exception { + NoRtfFile(); +}; + /// @brief Unsupported crypto algorithm exception struct UnsupportedCryptoAlgorithm final : Exception { UnsupportedCryptoAlgorithm(); diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 61c6ecdd..034328e9 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -395,7 +395,7 @@ constexpr std::array table{ .color_scheme = true}}, // Recognised by magic so a caller can name the type, but there is no - // decoder behind either of these. + // decoder behind it. Row{FileType::word_perfect, "wpd"sv, wpd_extensions, @@ -403,13 +403,17 @@ constexpr std::array table{ FileCategory::document, DocumentType::unknown, {.detect_by_content = true}}, + Row{FileType::rich_text_format, "rtf"sv, rtf_extensions, rtf_mimetypes, FileCategory::document, - DocumentType::unknown, - {.detect_by_content = true}}, + DocumentType::text, + {.detect_by_content = true, + .open = true, + .translate_html = true, + .color_scheme = true}}, Row{FileType::portable_document_format, "pdf"sv, diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 68a4e260..364a0bb1 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -112,6 +113,16 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoPdfFile(); } + if (as == FileType::rich_text_format) { + ODR_VERBOSE(logger, "open as rtf"); + try { + return std::make_unique(file); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as rtf"); + } + throw NoRtfFile(); + } + if (as == FileType::starview_metafile) { ODR_VERBOSE(logger, "open as svm"); try { @@ -396,6 +407,10 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "open as pdf"); return std::make_unique(file); } + if (file_type == FileType::rich_text_format) { + ODR_VERBOSE(logger, "open as rtf"); + return std::make_unique(file); + } if (file_type == FileType::starview_metafile) { ODR_VERBOSE(logger, "open as svm"); return std::make_unique(file); @@ -565,6 +580,9 @@ open_strategy::open_document_file(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + } else if (file_type == FileType::rich_text_format) { + ODR_VERBOSE(logger, "open as rtf"); + return std::make_unique(file); } ODR_ERROR(logger, "unsupported file type for document file " diff --git a/src/odr/internal/rtf/AGENTS.md b/src/odr/internal/rtf/AGENTS.md new file mode 100644 index 00000000..46ce161c --- /dev/null +++ b/src/odr/internal/rtf/AGENTS.md @@ -0,0 +1,97 @@ +# AGENTS.md — `internal/rtf` + +Rich Text Format, read as a text document. Read [`PLAN.md`](PLAN.md) first: it +carries the staging, the decisions taken up front and what each later stage +owes. This file records what stage 1 actually built and where it deviates. + +Spec references are the RTF Specification 1.9.1 (March 2008). It has no section +numbers, so cite the heading and the control word — *Conventions of an RTF +Reader*, *Control Word*, *Table Definitions*. Not `[MS-OXRTFEX]` / +`[MS-OXRTFCP]`, which are different documents and out of scope. + +## Shape + +``` +bytes ─▶ Tokenizer ─▶ TreeBuilder ─▶ ElementRegistry ─▶ Document ─▶ RtfFile + (rtf_tokenizer) (rtf_parser) (rtf_document) (rtf_file) +``` + +| File | What | +|------|------| +| `rtf_token.hpp` | The `Token` variant the tokenizer yields. | +| `rtf_tokenizer.*` | Bytes → tokens. Knows nothing about groups or destinations. | +| `rtf_state.*` | The group stack: `{` saves, `}` restores. | +| `rtf_parser.*` | `parse_tree` — tokens → `root → paragraph → (text \| line break)`. | +| `rtf_element_registry.*` | The flat registry, copied from `oldms/text` minus its style index. | +| `rtf_document.*` | `internal::Document` + the element adapter. | +| `rtf_file.*` | `abstract::DocumentFile`; validates the magic, hands out the document. | + +There is no filesystem: an rtf is one byte stream, so `internal::Document` gets +a null `ReadableFilesystem` the way `csv` does. + +## What stage 1 decodes + +Paragraph structure and text: `\par`, `\line`, `\tab`, `\page`, `\sect`, the +literal-character control words (`\emdash`, `\bullet`, the quotes, …), the +escapes `\\` `\{` `\}` `\~` `\_` `\-`, `\'hh` in the run's encoding, and `\uN` +including surrogate pairs. The encoding comes from `\ansi` / `\mac` / `\pc` / +`\pca` / `\ansicpgN` through `internal/encoding`. + +Everything else is ignored, which is the spec's own rule for a reader meeting a +control word it does not know. Character and paragraph formatting, tables, +pictures and lists are stages 2–5 in `PLAN.md`. + +## Rules worth knowing before touching this + +- **Leniency is the spec here, and does not violate the root `AGENTS.md` + fail-fast rule.** Unknown control words are ignored, `{\*` groups whose + destination we do not implement are discarded, an unmatched `}` is ignored. + What *does* throw: a group left open at EOF, an invalid hex digit after `\'`, + a `\binN` running past EOF, a trailing `\`, and nesting past `State`'s depth + bound. +- **`\binN` is read by the tokenizer, not the parser.** Its payload is raw + bytes that may contain braces and backslashes, so a brace-counting scan over + them would desync the group nesting — including inside a group being + discarded. That is why skipping an ignorable destination still tokenizes. +- **Text is bytes until the run ends.** `\'hh` yields one *byte*: in a + double-byte run two consecutive escapes are one character. The accumulator is + `m_bytes` plus the run's `TextEncoding`, decoded through `encoding::to_utf8` + only at a flush. Decoding per escape corrupts every multibyte run. +- **`\uN` is a UTF-16 code unit, signed.** `U+F020` arrives as `\u-4064` (fold + with `+ 65536` *before* the surrogate test), and anything above the BMP + arrives as a surrogate pair across two `\uN`. A high surrogate is held pending + and combined with the low one; unpaired, it becomes U+FFFD. +- **`\ucN` counts control words and symbols as one character each**, strictly, + a `\binN` and its payload included. Real writers always emit the fallback, so + the strict reading costs nothing and is what the spec says. A group boundary + cancels a pending skip. +- **A `{\*`-marked destination needs no entry in the discard table**; the `\*` + control symbol discards whatever follows it. The table in `rtf_parser.cpp` is + only for destinations that are *not* marked — `\fonttbl`, `\colortbl`, + `\info`, `\pict`, and notably `\nonshppict`, the unmarked twin of + `{\*\shppict}` that would otherwise emit every image a second time. + +## Deviations from `PLAN.md` + +- **`HexEscape` is its own token**, not folded into `Text`. The plan's variant + had no place to put the decoded byte of a `\'hh`, and `\ucN` counts an escape + as one character where a text run counts bytes. +- **`\page` emits `ElementType::page_break`** (a child of root, as `oldms/text` + does) even though the html renderer ignores that type today. +- **`\cell` renders as a tab and `\row` as a paragraph end.** The plan defers + tables to stage 4; until then this keeps table text readable rather than + running it together. +- **An undecodable run degrades per byte**, not per run: ascii passes through + and only bytes ≥ 0x80 become U+FFFD. The plan said the whole run degrades, + which loses the ascii skeleton for no reason. + +## Testing + +Everything is inline string literals — an rtf fragment is readable in a raw +string, and there is no `.rtf` anywhere under `test/data`. `rtf_tokenizer_test` +covers the delimiter rules token by token; `rtf_document_test` runs +`parse_tree` and flattens the tree to one line (`P(…)`, `|`, `PB`). + +A render test against a real fixture needs a file committed to +`test/data/input` plus the reference-output regen, and is owed once stage 5 +lands a picture that cannot be written inline. diff --git a/src/odr/internal/rtf/PLAN.md b/src/odr/internal/rtf/PLAN.md index b82a859d..7f6a7ae4 100644 --- a/src/odr/internal/rtf/PLAN.md +++ b/src/odr/internal/rtf/PLAN.md @@ -5,12 +5,15 @@ it honest as stages land. ## Today -Nothing decodes. `FileType::rich_text_format` exists (`file.hpp:62`, under the -"Detection only" comment), `magic.cpp:123` matches `7B 5C 72 74 66 31` -(`{\rtf1`), and `file_type_table.cpp:390` carries a row with `rtf` extensions, -three mime types, `FileCategory::document`, `DocumentType::unknown` and -`{.detect_by_content = true}`. `open_strategy::open_file` has no branch for it, -so `odr::open` on an rtf reaches the fallthrough and throws `UnknownFileType`. +**Stage 1 has landed.** An rtf opens as a text document and renders: the +tokenizer, the group/destination machinery and plain text with paragraphs, line +breaks, tabs and the full `\'hh` / `\uN` decoding. The table row now declares +`DocumentType::text` and `{.open, .translate_html, .color_scheme}`, and +`open_strategy` has its three branches. See [`AGENTS.md`](AGENTS.md) for what +was built and where it deviates from what follows. + +Stages 2–5 below are untouched: no character or paragraph formatting, no page +layout, no tables, no pictures. The public enum is already mirrored by the bindings (`bind_file.cpp:39`, `ODRFile.mm:43`), so **no binding work is needed for any stage below** — the @@ -132,7 +135,7 @@ it rather than inventing a shared one. --- -## Stage 1 — plumbing, tokenizer, plain text +## Stage 1 — plumbing, tokenizer, plain text — **done** The narrowest thing that renders. No formatting at all beyond paragraph structure, so the tokenizer and the group machinery can be proven before diff --git a/src/odr/internal/rtf/rtf_document.cpp b/src/odr/internal/rtf/rtf_document.cpp new file mode 100644 index 00000000..21588ef0 --- /dev/null +++ b/src/odr/internal/rtf/rtf_document.cpp @@ -0,0 +1,189 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace odr::internal::rtf { + +namespace { +std::unique_ptr +create_element_adapter(const ElementRegistry ®istry); +} + +Document::Document(const abstract::File &file) + : internal::Document(FileType::rich_text_format, DocumentType::text, + nullptr) { + const std::unique_ptr in = file.stream(); + m_root_element = parse_tree(m_element_registry, *in); + + m_element_adapter = create_element_adapter(m_element_registry); +} + +const ElementRegistry &Document::element_registry() const { + return m_element_registry; +} + +bool Document::is_editable() const noexcept { return false; } + +bool Document::is_savable(const bool encrypted) const noexcept { + (void)encrypted; + return false; +} + +void Document::save(const Path &path) const { + (void)path; + throw UnsupportedOperation(); +} + +void Document::save(const Path &path, const char *password) const { + (void)path; + (void)password; + throw UnsupportedOperation(); +} + +namespace { + +class ElementAdapter final : public abstract::ElementAdapter, + public abstract::TextRootAdapter, + public abstract::LineBreakAdapter, + public abstract::ParagraphAdapter, + public abstract::TextAdapter { +public: + explicit ElementAdapter(const ElementRegistry ®istry) + : m_registry(®istry) {} + + [[nodiscard]] ElementType + element_type(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).type; + } + + [[nodiscard]] ElementIdentifier + element_parent(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).parent_id; + } + [[nodiscard]] ElementIdentifier + element_first_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).first_child_id; + } + [[nodiscard]] ElementIdentifier + element_last_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).last_child_id; + } + [[nodiscard]] ElementIdentifier + element_previous_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).previous_sibling_id; + } + [[nodiscard]] ElementIdentifier + element_next_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).next_sibling_id; + } + + [[nodiscard]] bool + element_is_unique(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_self_locatable(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_editable(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] DocumentPath + element_document_path(const ElementIdentifier element_id) const override { + return util::document::extract_path(*this, element_id, null_element_id); + } + [[nodiscard]] ElementIdentifier + element_navigate_path(const ElementIdentifier element_id, + const DocumentPath &path) const override { + return util::document::navigate_path(*this, element_id, path); + } + + [[nodiscard]] const TextRootAdapter * + text_root_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::root ? this : nullptr; + } + [[nodiscard]] const LineBreakAdapter * + line_break_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::line_break ? this : nullptr; + } + [[nodiscard]] const ParagraphAdapter * + paragraph_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::paragraph ? this : nullptr; + } + [[nodiscard]] const TextAdapter * + text_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::text ? this : nullptr; + } + + [[nodiscard]] PageLayout + text_root_page_layout(const ElementIdentifier element_id) const override { + // `\paperwN` and the margins arrive with `PLAN.md` stage 3 + (void)element_id; + return {}; + } + [[nodiscard]] ElementIdentifier text_root_first_master_page( + const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TextStyle + line_break_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] ParagraphStyle + paragraph_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] TextStyle + paragraph_text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] std::string + text_content(const ElementIdentifier element_id) const override { + return m_registry->text_element_at(element_id).text; + } + void text_set_content(const ElementIdentifier element_id, + const std::string &text) const override { + (void)element_id; + (void)text; + throw UnsupportedOperation(); + } + [[nodiscard]] TextStyle + text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + +private: + const ElementRegistry *m_registry{nullptr}; +}; + +std::unique_ptr +create_element_adapter(const ElementRegistry ®istry) { + return std::make_unique(registry); +} + +} // namespace + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_document.hpp b/src/odr/internal/rtf/rtf_document.hpp new file mode 100644 index 00000000..62b6c571 --- /dev/null +++ b/src/odr/internal/rtf/rtf_document.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace odr::internal::abstract { +class File; +} + +namespace odr::internal::rtf { + +/// An rtf as a text document. Not backed by a filesystem: the whole file is +/// one byte stream, parsed into the element registry at construction. +class Document final : public internal::Document { +public: + explicit Document(const abstract::File &file); + + [[nodiscard]] const ElementRegistry &element_registry() const; + + [[nodiscard]] bool is_editable() const noexcept override; + [[nodiscard]] bool is_savable(bool encrypted) const noexcept override; + + void save(const Path &path) const override; + void save(const Path &path, const char *password) const override; + +private: + ElementRegistry m_element_registry; +}; + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_element_registry.cpp b/src/odr/internal/rtf/rtf_element_registry.cpp new file mode 100644 index 00000000..31df8dbb --- /dev/null +++ b/src/odr/internal/rtf/rtf_element_registry.cpp @@ -0,0 +1,94 @@ +#include + +#include + +namespace odr::internal::rtf { + +void ElementRegistry::clear() noexcept { + m_elements.clear(); + m_texts.clear(); +} + +std::size_t ElementRegistry::size() const noexcept { return m_elements.size(); } + +std::tuple +ElementRegistry::create_element(const ElementType type) { + Element &element = m_elements.emplace_back(); + ElementIdentifier element_id = m_elements.size(); + element.type = type; + return {element_id, element}; +} + +std::tuple +ElementRegistry::create_text_element() { + const auto &[element_id, element] = create_element(ElementType::text); + auto [it, success] = m_texts.emplace(element_id, Text{}); + return {element_id, element, it->second}; +} + +ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) { + check_element_id(id); + return m_elements.at(id - 1); +} + +ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) { + check_text_id(id); + return m_texts.at(id); +} + +const ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) const { + check_element_id(id); + return m_elements.at(id - 1); +} + +const ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) const { + check_text_id(id); + return m_texts.at(id); +} + +void ElementRegistry::append_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id) { + check_element_id(parent_id); + check_element_id(child_id); + if (element_at(child_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_child: child already has a parent"); + } + + const ElementIdentifier previous_sibling_id = + element_at(parent_id).last_child_id; + + element_at(child_id).parent_id = parent_id; + element_at(child_id).previous_sibling_id = previous_sibling_id; + + if (element_at(parent_id).first_child_id == null_element_id) { + element_at(parent_id).first_child_id = child_id; + } else { + element_at(previous_sibling_id).next_sibling_id = child_id; + } + element_at(parent_id).last_child_id = child_id; +} + +void ElementRegistry::check_element_id(const ElementIdentifier id) const { + if (id == null_element_id) { + throw std::out_of_range("ElementRegistry::check_id: null identifier"); + } + if (id - 1 >= m_elements.size()) { + throw std::out_of_range( + "ElementRegistry::check_id: identifier out of range"); + } +} + +void ElementRegistry::check_text_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_texts.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_element_registry.hpp b/src/odr/internal/rtf/rtf_element_registry.hpp new file mode 100644 index 00000000..cf1f8b01 --- /dev/null +++ b/src/odr/internal/rtf/rtf_element_registry.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace odr::internal::rtf { + +class ElementRegistry final { +public: + struct Element final { + ElementIdentifier parent_id{null_element_id}; + ElementIdentifier first_child_id{null_element_id}; + ElementIdentifier last_child_id{null_element_id}; + ElementIdentifier previous_sibling_id{null_element_id}; + ElementIdentifier next_sibling_id{null_element_id}; + ElementType type{ElementType::none}; + }; + + struct Text final { + std::string text; + }; + + void clear() noexcept; + + [[nodiscard]] std::size_t size() const noexcept; + + std::tuple create_element(ElementType type); + std::tuple create_text_element(); + + [[nodiscard]] Element &element_at(ElementIdentifier id); + [[nodiscard]] Text &text_element_at(ElementIdentifier id); + + [[nodiscard]] const Element &element_at(ElementIdentifier id) const; + [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; + + void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + +private: + std::vector m_elements; + std::unordered_map m_texts; + + void check_element_id(ElementIdentifier id) const; + void check_text_id(ElementIdentifier id) const; +}; + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_file.cpp b/src/odr/internal/rtf/rtf_file.cpp new file mode 100644 index 00000000..eb9d5990 --- /dev/null +++ b/src/odr/internal/rtf/rtf_file.cpp @@ -0,0 +1,47 @@ +#include + +#include + +#include +#include + +#include + +namespace odr::internal::rtf { + +RtfFile::RtfFile(std::shared_ptr file) + : m_file{std::move(file)} { + if (magic::file_type(*m_file) != FileType::rich_text_format) { + throw NoRtfFile(); + } +} + +std::shared_ptr RtfFile::file() const noexcept { + return m_file; +} + +FileType RtfFile::file_type() const noexcept { + return FileType::rich_text_format; +} + +std::string_view RtfFile::mimetype() const noexcept { + return "application/rtf"; +} + +FileMeta RtfFile::file_meta() const noexcept { + FileMeta result; + result.type = file_type(); + result.mimetype = mimetype(); + result.document_type = document_type(); + return result; +} + +DocumentType RtfFile::document_type() const { return DocumentType::text; } + +bool RtfFile::is_decodable() const noexcept { return true; } + +std::shared_ptr RtfFile::document() const { + return std::make_shared(*m_file); +} + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_file.hpp b/src/odr/internal/rtf/rtf_file.hpp new file mode 100644 index 00000000..ca160dd8 --- /dev/null +++ b/src/odr/internal/rtf/rtf_file.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace odr::internal::rtf { + +class RtfFile final : public abstract::DocumentFile { +public: + /// @throws NoRtfFile if the bytes do not open with `{\rtf1`. + explicit RtfFile(std::shared_ptr file); + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] DocumentType document_type() const override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + [[nodiscard]] std::shared_ptr document() const override; + +private: + std::shared_ptr m_file; +}; + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_parser.cpp b/src/odr/internal/rtf/rtf_parser.cpp new file mode 100644 index 00000000..e61945d9 --- /dev/null +++ b/src/odr/internal/rtf/rtf_parser.cpp @@ -0,0 +1,566 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +using namespace rtf; + +constexpr char32_t replacement_character = 0xfffd; + +/// Destinations the spec defines that carry no body text at this fidelity. +/// `{\*`-marked ones need no entry — the `\*` control symbol discards those +/// whatever their destination is. +const std::unordered_set &discarded_destinations() { + static const std::unordered_set destinations{ + // header tables + "fonttbl", "filetbl", "colortbl", "stylesheet", "listtable", + "listoverridetable", "revtbl", "rsidtbl", "info", + // deferred document parts, see `PLAN.md` + "header", "headerl", "headerr", "headerf", "footer", "footerl", "footerr", + "footerf", "footnote", "annotation", "atnauthor", "atndate", "atnid", + "atnparent", "atnref", "atntime", "atnicn", + // indexes and bookmarks: markers, not text + "xe", "tc", "txe", "bkmkstart", "bkmkend", + // a field instruction is normally `{\*\fldinst}`, but not every writer + // marks it; `\fldrslt` is left to flow, which is the cached-result-only + // position `oldms/text` takes too + "fldinst", + // pictures and embedded objects, see `PLAN.md` stage 5. `\nonshppict` + // is the un-marked twin of `{\*\shppict}` and would otherwise emit + // every image a second time + "pict", "nonshppict", "object", "objdata", "result", "do", "shptxt", + // font and style descriptors + "falt", "fname", "ffname", "panose", "keycode", + // math and drawing + "mmath", "factoidname", "datafield", "template", "private"}; + return destinations; +} + +/// Control words that stand for exactly one character. +const std::unordered_map &literal_characters() { + static const std::unordered_map characters{ + {"emdash", 0x2014}, {"endash", 0x2013}, {"emspace", 0x2003}, + {"enspace", 0x2002}, {"qmspace", 0x2005}, {"bullet", 0x2022}, + {"lquote", 0x2018}, {"rquote", 0x2019}, {"ldblquote", 0x201c}, + {"rdblquote", 0x201d}, {"ltrmark", 0x200e}, {"rtlmark", 0x200f}, + {"zwj", 0x200d}, {"zwnj", 0x200c}, {"zwbo", 0x200b}, + {"zwnbo", 0xfeff}}; + return characters; +} + +/// The `\ansicpgN` code page as a @ref TextEncoding, or windows-1252 for one +/// this build has no row for. +TextEncoding encoding_by_codepage(const std::int32_t codepage) { + switch (codepage) { + case 866: + return TextEncoding::ibm866; + case 874: + return TextEncoding::windows_874; + case 932: + return TextEncoding::shift_jis; + case 936: + return TextEncoding::gb18030; + case 949: + return TextEncoding::euc_kr; + case 950: + return TextEncoding::big5; + case 1250: + return TextEncoding::windows_1250; + case 1251: + return TextEncoding::windows_1251; + case 1252: + return TextEncoding::windows_1252; + case 1253: + return TextEncoding::windows_1253; + case 1254: + return TextEncoding::windows_1254; + case 1255: + return TextEncoding::windows_1255; + case 1256: + return TextEncoding::windows_1256; + case 1257: + return TextEncoding::windows_1257; + case 1258: + return TextEncoding::windows_1258; + case 10000: + return TextEncoding::macintosh; + case 10007: + return TextEncoding::x_mac_cyrillic; + case 20866: + return TextEncoding::koi8_r; + case 21866: + return TextEncoding::koi8_u; + case 28591: + return TextEncoding::iso_8859_1; + case 28592: + return TextEncoding::iso_8859_2; + case 28593: + return TextEncoding::iso_8859_3; + case 28594: + return TextEncoding::iso_8859_4; + case 28595: + return TextEncoding::iso_8859_5; + case 28596: + return TextEncoding::iso_8859_6; + case 28597: + return TextEncoding::iso_8859_7; + case 28598: + return TextEncoding::iso_8859_8; + case 28600: + return TextEncoding::iso_8859_10; + case 28603: + return TextEncoding::iso_8859_13; + case 28604: + return TextEncoding::iso_8859_14; + case 28605: + return TextEncoding::iso_8859_15; + case 28606: + return TextEncoding::iso_8859_16; + case 50220: + case 50221: + case 50222: + return TextEncoding::iso_2022_jp; + case 50225: + return TextEncoding::iso_2022_kr; + case 51932: + return TextEncoding::euc_jp; + case 65001: + return TextEncoding::utf8; + default: + return TextEncoding::windows_1252; + } +} + +bool is_high_surrogate(const char16_t unit) { + return unit >= 0xd800 && unit <= 0xdbff; +} + +bool is_low_surrogate(const char16_t unit) { + return unit >= 0xdc00 && unit <= 0xdfff; +} + +class TreeBuilder final { +public: + TreeBuilder(ElementRegistry ®istry, std::istream &in) + : m_registry{®istry}, m_tokenizer{in} {} + + ElementIdentifier parse(); + +private: + ElementRegistry *m_registry{nullptr}; + Tokenizer m_tokenizer; + State m_state; + + ElementIdentifier m_root{null_element_id}; + ElementIdentifier m_paragraph{null_element_id}; + + /// The run so far, still in @ref m_encoding — `\'hh` yields a *byte*, so two + /// escapes can be one character and decoding per escape would corrupt every + /// multibyte run. + std::string m_bytes; + /// The current paragraph's text so far, already utf-8. + std::string m_text; + TextEncoding m_encoding{TextEncoding::windows_1252}; + + /// A `\uN` high surrogate waiting for the low one that completes it. + char16_t m_high_surrogate{0}; + /// `\ucN` fallback characters still to be skipped. + std::int32_t m_skip{0}; + + /// Whether this token is one of the `\ucN` fallback characters, which counts + /// it as consumed. Any control word or symbol counts as one character, as + /// does a `\binN` together with its payload. + bool consume_skip(); + + void handle_control_word(const ControlWord &word); + void handle_control_symbol(const ControlSymbol &symbol); + void handle_text(std::string_view bytes); + + void unicode_character(std::int32_t value); + void append_character(char32_t character); + void append_text(std::string_view text); + + void resolve_surrogate(); + void flush_bytes(); + /// End the run: an unpaired surrogate becomes U+FFFD, the bytes are decoded. + void flush_run(); + + void ensure_paragraph(); + void flush_text(); + void end_paragraph(); + void line_break(); + void page_break(); + void finish(); +}; + +ElementIdentifier TreeBuilder::parse() { + auto [root_id, _] = m_registry->create_element(ElementType::root); + m_root = root_id; + + while (true) { + const Token token = m_tokenizer.read_token(); + + if (std::holds_alternative(token)) { + if (m_state.depth() > 1) { + throw std::runtime_error("rtf: group left open at end of file"); + } + break; + } + if (std::holds_alternative(token)) { + flush_run(); + m_skip = 0; + m_state.save(); + continue; + } + if (std::holds_alternative(token)) { + flush_run(); + m_skip = 0; + m_state.restore(); + continue; + } + if (const auto *word = std::get_if(&token)) { + if (!consume_skip()) { + handle_control_word(*word); + } + continue; + } + if (const auto *symbol = std::get_if(&token)) { + if (!consume_skip()) { + handle_control_symbol(*symbol); + } + continue; + } + if (const auto *hex = std::get_if(&token)) { + if (!consume_skip() && !m_state.current().discard) { + m_bytes.push_back(hex->byte); + } + continue; + } + if (std::holds_alternative(token)) { + // no destination reads the payload yet; it counts as one skippable + // character together with its `\binN` + consume_skip(); + continue; + } + if (const auto *text = std::get_if(&token)) { + handle_text(text->bytes); + continue; + } + } + + finish(); + return m_root; +} + +bool TreeBuilder::consume_skip() { + if (m_skip <= 0) { + return false; + } + --m_skip; + return true; +} + +void TreeBuilder::handle_control_word(const ControlWord &word) { + const std::string &name = word.name; + + if (discarded_destinations().contains(name)) { + m_state.current().discard = true; + return; + } + + // The document encoding is set in the header, which is never discarded, but + // it is read before the discard check so a header written inside a group we + // drop still lands. + if (name == "ansi") { + m_encoding = TextEncoding::windows_1252; + return; + } + if (name == "mac") { + m_encoding = TextEncoding::macintosh; + return; + } + if (name == "pc" || name == "pca") { + // IBM code pages 437 and 850, neither of which `internal/encoding` has + m_encoding = TextEncoding::windows_1252; + return; + } + if (name == "ansicpg") { + m_encoding = encoding_by_codepage(word.parameter.value_or(1252)); + return; + } + + if (name == "uc") { + m_state.current().uc = std::max(0, word.parameter.value_or(1)); + return; + } + if (name == "u") { + unicode_character(word.parameter.value_or(0)); + return; + } + + if (m_state.current().discard) { + return; + } + + if (name == "par" || name == "sect") { + end_paragraph(); + return; + } + if (name == "page") { + page_break(); + return; + } + if (name == "line" || name == "softline") { + line_break(); + return; + } + if (name == "tab") { + append_text("\t"); + return; + } + // Without the table reconstruction of `PLAN.md` stage 4, a cell boundary + // still reads as a column break and a row as a paragraph. + if (name == "cell" || name == "nestcell") { + append_text("\t"); + return; + } + if (name == "row" || name == "nestrow") { + end_paragraph(); + return; + } + + if (const auto it = literal_characters().find(name); + it != literal_characters().end()) { + append_character(it->second); + return; + } + + // an unknown control word is ignored (*Conventions of an RTF Reader*) +} + +void TreeBuilder::handle_control_symbol(const ControlSymbol &symbol) { + switch (symbol.symbol) { + case '*': + // an ignorable destination: everything to the matching `}` is dropped + m_state.current().discard = true; + break; + case '\\': + case '{': + case '}': + if (!m_state.current().discard) { + m_bytes.push_back(symbol.symbol); + } + break; + case '~': + append_character(0x00a0); // non-breaking space + break; + case '_': + append_character(0x2011); // non-breaking hyphen + break; + case '\r': + case '\n': + // `\` and `\` are `\par` (*Conventions of an RTF Reader*) + if (!m_state.current().discard) { + end_paragraph(); + } + break; + case '-': // optional hyphen: nothing is rendered where it does not break + case ':': // subentry of an index entry + case '|': // formula character + default: + break; + } +} + +void TreeBuilder::handle_text(std::string_view bytes) { + if (m_skip > 0) { + const auto skipped = + std::min(static_cast(m_skip), bytes.size()); + m_skip -= static_cast(skipped); + bytes.remove_prefix(skipped); + } + if (bytes.empty() || m_state.current().discard) { + return; + } + m_bytes.append(bytes); +} + +void TreeBuilder::unicode_character(const std::int32_t value) { + m_skip = m_state.current().uc; + + if (m_state.current().discard) { + return; + } + + // Content between the two halves breaks the pair, and the bytes have to + // come out after the replacement character that ends the run. + if (!m_bytes.empty()) { + flush_run(); + } + + // the parameter is a signed 16-bit code *unit*, so U+F020 arrives as -4064 + std::int32_t value_folded = value; + if (value_folded < 0) { + value_folded += 0x10000; + } + if (value_folded < 0 || value_folded > 0xffff) { + resolve_surrogate(); + util::string::append_c32(replacement_character, m_text); + return; + } + + const auto unit = static_cast(value_folded); + + if (m_high_surrogate != 0) { + if (is_low_surrogate(unit)) { + const auto character = static_cast( + 0x10000 + ((m_high_surrogate - 0xd800) << 10) + (unit - 0xdc00)); + m_high_surrogate = 0; + util::string::append_c32(character, m_text); + return; + } + resolve_surrogate(); + } + + if (is_high_surrogate(unit)) { + m_high_surrogate = unit; + return; + } + if (is_low_surrogate(unit)) { + util::string::append_c32(replacement_character, m_text); + return; + } + util::string::append_c32(unit, m_text); +} + +void TreeBuilder::append_character(const char32_t character) { + if (m_state.current().discard) { + return; + } + flush_run(); + util::string::append_c32(character, m_text); +} + +void TreeBuilder::append_text(const std::string_view text) { + if (m_state.current().discard) { + return; + } + flush_run(); + m_text.append(text); +} + +void TreeBuilder::resolve_surrogate() { + if (m_high_surrogate == 0) { + return; + } + m_high_surrogate = 0; + util::string::append_c32(replacement_character, m_text); +} + +void TreeBuilder::flush_bytes() { + if (m_bytes.empty()) { + return; + } + if (text_encoding_is_decodable(m_encoding)) { + m_text += encoding::to_utf8(m_bytes, m_encoding); + } else { + // A named-but-not-decoded multibyte encoding. The ascii skeleton is the + // same in all of them, and a writer emitting cjk generally emits `\uN` + // alongside, which is decoded whatever the run's encoding. + for (const char byte : m_bytes) { + if (static_cast(byte) < 0x80) { + m_text.push_back(byte); + } else { + util::string::append_c32(replacement_character, m_text); + } + } + } + m_bytes.clear(); +} + +void TreeBuilder::flush_run() { + resolve_surrogate(); + flush_bytes(); +} + +void TreeBuilder::ensure_paragraph() { + if (m_paragraph != null_element_id) { + return; + } + auto [id, _] = m_registry->create_element(ElementType::paragraph); + m_registry->append_child(m_root, id); + m_paragraph = id; +} + +void TreeBuilder::flush_text() { + if (m_text.empty()) { + return; + } + auto [id, element, entry] = m_registry->create_text_element(); + entry.text = std::move(m_text); + m_text.clear(); + m_registry->append_child(m_paragraph, id); +} + +void TreeBuilder::end_paragraph() { + flush_run(); + ensure_paragraph(); + flush_text(); + m_paragraph = null_element_id; +} + +void TreeBuilder::line_break() { + flush_run(); + ensure_paragraph(); + flush_text(); + auto [id, _] = m_registry->create_element(ElementType::line_break); + m_registry->append_child(m_paragraph, id); +} + +void TreeBuilder::page_break() { + end_paragraph(); + auto [id, _] = m_registry->create_element(ElementType::page_break); + m_registry->append_child(m_root, id); +} + +void TreeBuilder::finish() { + flush_run(); + // paragraphs open lazily, so the trailing `\par` adds no empty one + if (m_paragraph == null_element_id && m_text.empty()) { + return; + } + ensure_paragraph(); + flush_text(); +} + +} // namespace + +ElementIdentifier rtf::parse_tree(ElementRegistry ®istry, std::istream &in) { + return TreeBuilder(registry, in).parse(); +} + +} // namespace odr::internal diff --git a/src/odr/internal/rtf/rtf_parser.hpp b/src/odr/internal/rtf/rtf_parser.hpp new file mode 100644 index 00000000..2b27415d --- /dev/null +++ b/src/odr/internal/rtf/rtf_parser.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +#include + +namespace odr::internal::rtf { +class ElementRegistry; + +/// Parses the rtf byte stream into root → paragraph → (text | line break) +/// elements. Character and paragraph formatting, tables and pictures are not +/// modelled yet; see `PLAN.md`. +/// \return the root element id. +ElementIdentifier parse_tree(ElementRegistry ®istry, std::istream &in); + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_state.cpp b/src/odr/internal/rtf/rtf_state.cpp new file mode 100644 index 00000000..e03091cc --- /dev/null +++ b/src/odr/internal/rtf/rtf_state.cpp @@ -0,0 +1,28 @@ +#include + +#include + +namespace odr::internal::rtf { + +State::State() : m_stack(1) {} + +State::Group &State::current() { return m_stack.back(); } + +const State::Group &State::current() const { return m_stack.back(); } + +void State::save() { + if (m_stack.size() >= max_depth) { + throw std::runtime_error("rtf: group nesting too deep"); + } + m_stack.push_back(m_stack.back()); +} + +void State::restore() { + if (m_stack.size() > 1) { + m_stack.pop_back(); + } +} + +std::size_t State::depth() const noexcept { return m_stack.size(); } + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_state.hpp b/src/odr/internal/rtf/rtf_state.hpp new file mode 100644 index 00000000..a4bf64cf --- /dev/null +++ b/src/odr/internal/rtf/rtf_state.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +namespace odr::internal::rtf { + +/// The group stack of *Conventions of an RTF Reader*: `{` stores the current +/// state, `}` retrieves it. Never empty, so an unmatched `}` is ignored and +/// the state the document started from always remains. +class State final { +public: + struct Group final { + /// The group's text is not body text — a control destination, or a `{\*` + /// group whose destination we do not implement. + bool discard{false}; + /// `\ucN`: how many fallback characters follow each `\uN`. + std::int32_t uc{1}; + }; + + State(); + + [[nodiscard]] Group ¤t(); + [[nodiscard]] const Group ¤t() const; + + void save(); + /// An unmatched `}` is ignored. + void restore(); + + [[nodiscard]] std::size_t depth() const noexcept; + +private: + /// A group stack is heap, so this is not about stack overflow: it fails + /// fast on input that nests far past anything a writer produces. + static constexpr std::size_t max_depth = 1024; + + std::vector m_stack; +}; + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_token.hpp b/src/odr/internal/rtf/rtf_token.hpp new file mode 100644 index 00000000..aed02543 --- /dev/null +++ b/src/odr/internal/rtf/rtf_token.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +namespace odr::internal::rtf { + +/// `{` +struct GroupOpen final {}; +/// `}` +struct GroupClose final {}; +/// `\` plus ascii letters, with the signed parameter that followed them. +struct ControlWord final { + std::string name; + std::optional parameter; +}; +/// `\` plus one non-letter — `\\`, `\~`, `\*`, … Never `\'`. +struct ControlSymbol final { + char symbol{}; +}; +/// The one byte a `\'hh` escape stands for, still in the run's encoding. Its +/// own token because `\ucN` counts it as one character where a text run counts +/// bytes. +struct HexEscape final { + char byte{}; +}; +/// A literal run, still in the run's encoding. +struct Text final { + std::string bytes; +}; +/// The payload of a `\binN`, read raw — it may contain braces and backslashes. +struct Binary final { + std::string bytes; +}; +struct End final {}; + +using Token = std::variant; + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_tokenizer.cpp b/src/odr/internal/rtf/rtf_tokenizer.cpp new file mode 100644 index 00000000..4ecea622 --- /dev/null +++ b/src/odr/internal/rtf/rtf_tokenizer.cpp @@ -0,0 +1,191 @@ +#include + +#include +#include +#include + +namespace odr::internal::rtf { + +namespace { + +/// Only the ascii letters open a control word (*Control Word*); the locale +/// must not widen that. +bool is_letter(const char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +bool is_digit(const char c) { return c >= '0' && c <= '9'; } + +} // namespace + +Tokenizer::Tokenizer(std::istream &in) : m_in{&in}, m_sb{in.rdbuf()} { + // One-time stream preparation (flush tied streams, state check) for the raw + // streambuf reads below; a sentry's effects live entirely in its + // constructor, so it is not kept as state. + const std::istream::sentry se(in, true); +} + +Tokenizer::int_type Tokenizer::geti() { + const int_type c = m_sb->sgetc(); + if (c == eof) { + m_in->setstate(std::ios::eofbit); + } + return c; +} + +Tokenizer::char_type Tokenizer::bumpc() { + const int_type c = m_sb->sbumpc(); + if (c == eof) { + m_in->setstate(std::ios::eofbit); + throw std::runtime_error("unexpected stream exhaust"); + } + return static_cast(c); +} + +std::string Tokenizer::bumpnc(const std::size_t n) { + std::string result(n, '\0'); + if (const auto m = static_cast(n); + m_sb->sgetn(result.data(), m) != m) { + m_in->setstate(std::ios::eofbit); + throw std::runtime_error("unexpected stream exhaust"); + } + return result; +} + +std::uint8_t Tokenizer::hex_char_to_int(const char_type c) { + if (c >= '0' && c <= '9') { + return static_cast(c - '0'); + } + if (c >= 'a' && c <= 'f') { + return static_cast(c - 'a' + 10); + } + if (c >= 'A' && c <= 'F') { + return static_cast(c - 'A' + 10); + } + throw std::runtime_error("invalid hex digit"); +} + +Tokenizer::char_type Tokenizer::two_hex_to_char(const char_type first, + const char_type second) { + return static_cast((hex_char_to_int(first) << 4) | + hex_char_to_int(second)); +} + +Token Tokenizer::read_token() { + const int_type i = geti(); + if (i == eof) { + return End{}; + } + switch (static_cast(i)) { + case '{': + bumpc(); + return GroupOpen{}; + case '}': + bumpc(); + return GroupClose{}; + case '\\': + bumpc(); + return read_control(); + default: + return read_text(); + } +} + +Token Tokenizer::read_control() { + const int_type i = geti(); + if (i == eof) { + throw std::runtime_error("rtf: trailing backslash"); + } + + if (const auto c = static_cast(i); !is_letter(c)) { + bumpc(); + if (c == '\'') { + const char_type first = bumpc(); + const char_type second = bumpc(); + return HexEscape{two_hex_to_char(first, second)}; + } + // a control symbol takes no delimiter: a space after it is text + return ControlSymbol{c}; + } + + std::string name; + while (true) { + const int_type letter = geti(); + if (letter == eof || !is_letter(static_cast(letter))) { + break; + } + name.push_back(bumpc()); + } + + // The parameter's terminator is a delimiter under the same rule as the + // name's: a space is consumed, anything else stays unread. `\fs24 Text` + // therefore starts its text at `T`, and `\bin4 ` puts the payload right + // after the consumed space. + std::optional parameter; + if (const int_type delimiter = geti(); delimiter != eof) { + const auto d = static_cast(delimiter); + if (d == '-' || is_digit(d)) { + const bool negative = d == '-'; + if (negative) { + bumpc(); + } + std::int64_t value = 0; + std::size_t digits = 0; + while (digits < max_parameter_digits) { + const int_type digit = geti(); + if (digit == eof || !is_digit(static_cast(digit))) { + break; + } + value = value * 10 + (bumpc() - '0'); + ++digits; + } + if (digits > 0) { + if (negative) { + value = -value; + } + value = std::min( + std::max(value, + std::numeric_limits::min()), + std::numeric_limits::max()); + parameter = static_cast(value); + } + if (geti() == ' ') { + bumpc(); + } + } else if (d == ' ') { + bumpc(); + } + } + + // `\binN` is the one control word whose payload the tokenizer has to read + // itself: the bytes are raw, so a brace-counting scan over them would + // desync the group nesting. + if (name == "bin") { + const std::int32_t n = parameter.value_or(0); + return Binary{n > 0 ? bumpnc(static_cast(n)) : std::string()}; + } + + return ControlWord{std::move(name), parameter}; +} + +Token Tokenizer::read_text() { + std::string bytes; + while (true) { + const int_type i = geti(); + if (i == eof) { + break; + } + const auto c = static_cast(i); + if (c == '{' || c == '}' || c == '\\') { + break; + } + bumpc(); + // a bare line break is not text (*Conventions of an RTF Reader*) + if (c != '\r' && c != '\n') { + bytes.push_back(c); + } + } + return Text{std::move(bytes)}; +} + +} // namespace odr::internal::rtf diff --git a/src/odr/internal/rtf/rtf_tokenizer.hpp b/src/odr/internal/rtf/rtf_tokenizer.hpp new file mode 100644 index 00000000..99bed0be --- /dev/null +++ b/src/odr/internal/rtf/rtf_tokenizer.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace odr::internal::rtf { + +/// Pull-based token stream over the rtf byte grammar (*Conventions of an RTF +/// Reader*). Groups, destinations and encodings are the parser's business; +/// this only splits bytes into tokens. +class Tokenizer final { +public: + using char_type = std::streambuf::char_type; + using int_type = std::streambuf::int_type; + static constexpr int_type eof = std::streambuf::traits_type::eof(); + + explicit Tokenizer(std::istream &in); + + /// The next token; `End` once the stream is exhausted. + [[nodiscard]] Token read_token(); + + static std::uint8_t hex_char_to_int(char_type c); + static char_type two_hex_to_char(char_type first, char_type second); + +private: + /// *Control Word*: at most 10 digits, which also bounds the value the clamp + /// has to fit into `std::int32_t`. + static constexpr std::size_t max_parameter_digits = 10; + + int_type geti(); + char_type bumpc(); + std::string bumpnc(std::size_t n); + + /// With the leading `\` already consumed. + [[nodiscard]] Token read_control(); + [[nodiscard]] Token read_text(); + + std::istream *m_in{nullptr}; + std::streambuf *m_sb{nullptr}; +}; + +} // namespace odr::internal::rtf diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b516952..2379b93f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,9 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/rtf/rtf_document_test.cpp" + "src/internal/rtf/rtf_tokenizer_test.cpp" + "src/internal/odf/odf_table_test.cpp" "src/internal/oldms/doc_test.cpp" diff --git a/test/src/html_output_test.cpp b/test/src/html_output_test.cpp index eaf130fa..4160d605 100644 --- a/test/src/html_output_test.cpp +++ b/test/src/html_output_test.cpp @@ -86,7 +86,7 @@ TEST_P(HtmlOutputTests, html_meta) { ODR_INFO(logger, "Testing file: " << test_file.short_path << " output to: " << output_path); - // formats we cannot decode at all (wpd, rtf, md, …) plus the odd file we + // formats we cannot decode at all (wpd, md, …) plus the odd file we // classify but do not handle if (util::string::ends_with(test_file.short_path, ".sxw") || test_file.type == FileType::starview_metafile || diff --git a/test/src/internal/rtf/rtf_document_test.cpp b/test/src/internal/rtf/rtf_document_test.cpp new file mode 100644 index 00000000..4b83f05b --- /dev/null +++ b/test/src/internal/rtf/rtf_document_test.cpp @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace odr; +using namespace odr::internal; + +namespace { + +/// The parsed tree as one line: `P(…)` per paragraph, `|` for a line break and +/// `PB` for a page break. +std::string flatten(const std::string &content) { + rtf::ElementRegistry registry; + std::istringstream in(content); + const ElementIdentifier root = rtf::parse_tree(registry, in); + + std::string result; + for (ElementIdentifier child = registry.element_at(root).first_child_id; + child != null_element_id; + child = registry.element_at(child).next_sibling_id) { + if (registry.element_at(child).type == ElementType::page_break) { + result += "PB"; + continue; + } + result += "P("; + for (ElementIdentifier inner = registry.element_at(child).first_child_id; + inner != null_element_id; + inner = registry.element_at(inner).next_sibling_id) { + switch (registry.element_at(inner).type) { + case ElementType::text: + result += registry.text_element_at(inner).text; + break; + case ElementType::line_break: + result += "|"; + break; + default: + result += "?"; + break; + } + } + result += ")"; + } + return result; +} + +std::shared_ptr memory_file(const std::string &content) { + return std::make_shared(content); +} + +} // namespace + +TEST(RtfDocument, plain_paragraphs) { + EXPECT_EQ(flatten(R"({\rtf1\ansi Hello, World!\par})"), "P(Hello, World!)"); + EXPECT_EQ(flatten(R"({\rtf1\ansi one\par two\par})"), "P(one)P(two)"); +} + +TEST(RtfDocument, a_trailing_paragraph_mark_adds_no_empty_paragraph) { + EXPECT_EQ(flatten(R"({\rtf1\ansi one\par})"), "P(one)"); + // an explicit empty paragraph is real content and stays + EXPECT_EQ(flatten(R"({\rtf1\ansi one\par\par two\par})"), "P(one)P()P(two)"); +} + +TEST(RtfDocument, text_after_the_last_paragraph_mark) { + EXPECT_EQ(flatten(R"({\rtf1\ansi one\par two})"), "P(one)P(two)"); +} + +TEST(RtfDocument, line_break_tab_and_page_break) { + EXPECT_EQ(flatten(R"({\rtf1\ansi a\line b\par})"), "P(a|b)"); + EXPECT_EQ(flatten("{\\rtf1\\ansi a\\tab b\\par}"), "P(a\tb)"); + EXPECT_EQ(flatten(R"({\rtf1\ansi a\page b\par})"), "P(a)PBP(b)"); +} + +TEST(RtfDocument, a_line_break_control_symbol_is_a_paragraph) { + EXPECT_EQ(flatten("{\\rtf1\\ansi a\\\nb}"), "P(a)P(b)"); +} + +TEST(RtfDocument, character_control_words) { + EXPECT_EQ(flatten(R"({\rtf1\ansi a\emdash b\bullet\par})"), "P(a\xe2\x80\x94" + "b\xe2\x80\xa2)"); + EXPECT_EQ(flatten(R"({\rtf1\ansi a\~b\par})"), "P(a\xc2\xa0" + "b)"); +} + +TEST(RtfDocument, unknown_control_words_are_ignored) { + EXPECT_EQ(flatten(R"({\rtf1\ansi\b\i\fs28 bold\b0 and not\par})"), + "P(bold and not)"); +} + +TEST(RtfDocument, header_tables_are_not_body_text) { + EXPECT_EQ(flatten(R"({\rtf1\ansi\deff0)" + R"({\fonttbl{\f0\fnil Arial;}})" + R"({\colortbl;\red0\green0\blue0;})" + R"({\info{\title Ignored}})" + R"(Body\par})"), + "P(Body)"); +} + +TEST(RtfDocument, ignorable_destinations_are_skipped) { + EXPECT_EQ(flatten(R"({\rtf1\ansi{\*\generator Writer;}Body\par})"), + "P(Body)"); + // a field keeps its cached result and drops the instruction + EXPECT_EQ( + flatten(R"({\rtf1\ansi{\field{\*\fldinst PAGE }{\fldrslt 7}}\par})"), + "P(7)"); +} + +TEST(RtfDocument, a_binary_payload_does_not_desync_the_groups) { + EXPECT_EQ(flatten("{\\rtf1\\ansi{\\*\\x\\bin2 }}}Hello\\par}"), "P(Hello)"); +} + +TEST(RtfDocument, an_unmatched_group_close_is_ignored) { + EXPECT_EQ(flatten(R"({\rtf1\ansi a}}b\par})"), "P(ab)"); +} + +TEST(RtfDocument, a_group_left_open_throws) { + EXPECT_THROW(flatten(R"({\rtf1\ansi Hello)"), std::runtime_error); +} + +TEST(RtfDocument, hex_escapes_use_the_run_encoding) { + // windows-1252 by default + EXPECT_EQ(flatten(R"({\rtf1\ansi \'e4\par})"), "P(\xc3\xa4)"); + // and `\ansicpgN` overrides it + EXPECT_EQ(flatten(R"({\rtf1\ansi\ansicpg1251 \'e4\par})"), "P(\xd0\xb4)"); +} + +TEST(RtfDocument, unicode_escapes) { + // the parameter is signed, so U+F020 arrives as -4064 + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc1 \u-4064 ?\par})"), "P(\xef\x80\xa0)"); + // a non-bmp character is a surrogate pair, each half with its own fallback + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc1 \u-10179 ?\u-8704 ?\par})"), + "P(\xf0\x9f\x98\x80)"); + // an unpaired high surrogate is U+FFFD + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc1 \u-10179 ?A\par})"), "P(\xef\xbf\xbd" + "A)"); +} + +TEST(RtfDocument, uc_skips_a_control_word_as_one_character) { + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc1 \u65 \tab X\par})"), "P(AX)"); + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc2 \u65 ??X\par})"), "P(AX)"); + // `\ucN` is a character property, so a group restores it + EXPECT_EQ(flatten(R"({\rtf1\ansi\uc1 {\uc0 \u65 }\u66 ?X\par})"), "P(ABX)"); +} + +TEST(RtfDocument, table_rows_read_as_paragraphs) { + EXPECT_EQ(flatten(R"({\rtf1\ansi\trowd a\cell b\cell\row c\par})"), + "P(a\tb\t)P(c)"); +} + +TEST(RtfDocument, opens_through_the_file_layer) { + const rtf::RtfFile file(memory_file(R"({\rtf1\ansi Hello\par})")); + + EXPECT_EQ(file.file_type(), FileType::rich_text_format); + EXPECT_EQ(file.document_type(), DocumentType::text); + EXPECT_TRUE(file.is_decodable()); + + const Document document(file.document()); + EXPECT_EQ(document.file_type(), FileType::rich_text_format); + EXPECT_EQ(document.document_type(), DocumentType::text); + + const Element root = document.root_element(); + ASSERT_TRUE(root); + EXPECT_EQ(root.type(), ElementType::root); +} + +TEST(RtfDocument, something_else_is_no_rtf_file) { + EXPECT_THROW(rtf::RtfFile(memory_file("Hello, World!")), NoRtfFile); +} + +TEST(RtfDocument, translates_to_html) { + const auto file = std::make_shared( + memory_file(R"({\rtf1\ansi Hello, World!\par})")); + + std::ostringstream out; + html::translate(DecodedFile(file), {}).list_views().at(0).write_html(out); + + EXPECT_NE(out.str().find("Hello, World!"), std::string::npos); +} diff --git a/test/src/internal/rtf/rtf_tokenizer_test.cpp b/test/src/internal/rtf/rtf_tokenizer_test.cpp new file mode 100644 index 00000000..52ce1e92 --- /dev/null +++ b/test/src/internal/rtf/rtf_tokenizer_test.cpp @@ -0,0 +1,138 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace odr::internal; + +namespace { + +/// One token as a short string, so a whole stream reads as a vector literal: +/// `{`/`}` for the groups, `\name` (with `(parameter)`) for a control word, +/// `\c` for a control symbol, `'hh` for a `\'hh` escape, `[bytes]` for text +/// and `#bytes` for a `\binN` payload. +std::string describe(const rtf::Token &token) { + if (std::holds_alternative(token)) { + return "{"; + } + if (std::holds_alternative(token)) { + return "}"; + } + if (const auto *word = std::get_if(&token)) { + std::string result = "\\" + word->name; + if (word->parameter.has_value()) { + result += "(" + std::to_string(*word->parameter) + ")"; + } + return result; + } + if (const auto *symbol = std::get_if(&token)) { + return std::string("\\") + symbol->symbol; + } + if (const auto *hex = std::get_if(&token)) { + static constexpr std::array digits{'0', '1', '2', '3', '4', '5', + '6', '7', '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f'}; + const auto byte = static_cast(hex->byte); + return std::string("'") + digits.at(byte >> 4) + digits.at(byte & 0xf); + } + if (const auto *text = std::get_if(&token)) { + return "[" + text->bytes + "]"; + } + if (const auto *binary = std::get_if(&token)) { + return "#" + binary->bytes; + } + return ""; +} + +std::vector tokens(const std::string &content) { + std::istringstream in(content); + rtf::Tokenizer tokenizer(in); + + std::vector result; + while (true) { + const rtf::Token token = tokenizer.read_token(); + if (std::holds_alternative(token)) { + return result; + } + result.push_back(describe(token)); + } +} + +} // namespace + +TEST(RtfTokenizer, groups_and_text) { + EXPECT_EQ(tokens("{ab}"), (std::vector{"{", "[ab]", "}"})); +} + +TEST(RtfTokenizer, control_word_eats_its_delimiting_space) { + EXPECT_EQ(tokens("\\par Hello"), + (std::vector{"\\par", "[Hello]"})); +} + +TEST(RtfTokenizer, parameter_delimiter_follows_the_same_rule) { + // the space after the parameter is consumed, so the text starts at `T` + EXPECT_EQ(tokens("\\fs24 Text"), + (std::vector{"\\fs(24)", "[Text]"})); + // anything else is left unread + EXPECT_EQ(tokens("\\f0Hello"), + (std::vector{"\\f(0)", "[Hello]"})); +} + +TEST(RtfTokenizer, a_parameter_is_part_of_the_control_word) { + EXPECT_EQ(tokens("\\b0\\b "), (std::vector{"\\b(0)", "\\b"})); +} + +TEST(RtfTokenizer, a_control_symbol_takes_no_delimiter) { + // the space after `\~` is text, not a delimiter + EXPECT_EQ(tokens("\\~ x"), (std::vector{"\\~", "[ x]"})); + EXPECT_EQ(tokens("\\*\\foo"), (std::vector{"\\*", "\\foo"})); +} + +TEST(RtfTokenizer, escaped_braces_and_backslash) { + EXPECT_EQ(tokens("\\{\\}\\\\"), + (std::vector{"\\{", "\\}", "\\\\"})); +} + +TEST(RtfTokenizer, hex_escape) { + EXPECT_EQ(tokens("\\'41\\'E4"), (std::vector{"'41", "'e4"})); + // `\'7b` is a byte, never a group open + EXPECT_EQ(tokens("\\'7bx"), (std::vector{"'7b", "[x]"})); +} + +TEST(RtfTokenizer, an_invalid_hex_digit_throws) { + EXPECT_THROW(tokens("\\'4z"), std::runtime_error); +} + +TEST(RtfTokenizer, negative_parameter) { + EXPECT_EQ(tokens("\\u-4064 ?"), + (std::vector{"\\u(-4064)", "[?]"})); +} + +TEST(RtfTokenizer, a_bare_line_break_is_not_text) { + EXPECT_EQ(tokens("a\r\nb"), (std::vector{"[ab]"})); +} + +TEST(RtfTokenizer, binary_payload_is_read_raw) { + // the payload holds braces and a backslash, none of which are markup + EXPECT_EQ(tokens("\\bin5 }{\\ab}"), + (std::vector{"#}{\\ab", "}"})); +} + +TEST(RtfTokenizer, binary_without_a_parameter_is_empty) { + EXPECT_EQ(tokens("\\bin x"), (std::vector{"#", "[x]"})); +} + +TEST(RtfTokenizer, a_binary_payload_running_past_the_end_throws) { + EXPECT_THROW(tokens("\\bin9 ab"), std::runtime_error); +} + +TEST(RtfTokenizer, a_trailing_backslash_throws) { + EXPECT_THROW(tokens("ab\\"), std::runtime_error); +}