diff --git a/AGENTS.md b/AGENTS.md index d7b271006..bd5cc6dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | | `src/odr/internal/pdf/` | PDF (own parser). | +| `src/odr/internal/markdown/` | Markdown (CommonMark + GFM via md4c), decoded to a text document; see [`markdown/AGENTS.md`](src/odr/internal/markdown/AGENTS.md) + [`markdown/PLAN.md`](src/odr/internal/markdown/PLAN.md). | | `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | | `src/odr/internal/{csv,json,text,svm}/` | Smaller formats. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc85b981..ce32f050f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- `FileType::markdown` is reclassified from `FileCategory::text` to + `FileCategory::document` with `DocumentType::text`: `is_document_file()` + answers for it and `is_text_file()` no longer does, and + `file_category_by_file_type` says `document`. The row was + classification-only before, declaring no capabilities at all, so nothing + could hold a decoded markdown file to be broken. +- A markdown file opens as a text document — headings, paragraphs, lists, + block quotes, code blocks, emphasis, links, and GFM's tables, strikethrough + and task lists — and renders to html like any other document. CommonMark plus + the GitHub extensions, parsed with md4c (a new dependency). Raw html, images + and horizontal rules are not modelled yet. + Markdown has no signature, so it is never detected by content: open it as + `FileType::markdown` explicitly, or a `.md` still comes back as a text file. + ## v6.10.1 - 2026-08-21 - A linked image in a docx or xlsx (`embed_images = false`) is named relative diff --git a/CMakeLists.txt b/CMakeLists.txt index 04feb2197..a3c944dda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ endif () add_compile_options("$<$:/utf-8>") find_package(pugixml REQUIRED) +find_package(md4c REQUIRED) find_package(miniz REQUIRED) find_package(cryptopp REQUIRED) find_package(nlohmann_json REQUIRED) @@ -154,6 +155,12 @@ set(ODR_SOURCE_FILES "src/odr/internal/json/json_file.cpp" "src/odr/internal/json/json_util.cpp" + "src/odr/internal/markdown/markdown_document.cpp" + "src/odr/internal/markdown/markdown_element_registry.cpp" + "src/odr/internal/markdown/markdown_file.cpp" + "src/odr/internal/markdown/markdown_parser.cpp" + "src/odr/internal/markdown/markdown_style.cpp" + "src/odr/internal/odf/odf_crypto.cpp" "src/odr/internal/odf/odf_document.cpp" "src/odr/internal/odf/odf_element_registry.cpp" @@ -277,6 +284,7 @@ target_include_directories(odr target_link_libraries(odr PRIVATE pugixml::pugixml + md4c::md4c miniz::miniz cryptopp::cryptopp nlohmann_json::nlohmann_json diff --git a/README.md b/README.md index b796950be..444eaa5a3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ C++ library to visualize files, especially documents, in HTML. - [pdf](https://github.com/opendocument-app/OpenDocument.core/issues/108) - txt - json +- md (Markdown — CommonMark plus the GitHub extensions; never detected from its + bytes, so open it as `FileType::markdown` explicitly) - [zip](https://github.com/opendocument-app/OpenDocument.core/issues/109) - [cfb](https://github.com/opendocument-app/OpenDocument.core/issues/110) (Microsoft Compound File Binary File Format) - ttf / otf (font specimen pages) @@ -30,7 +32,6 @@ decoder, and opening one throws: - rtf - wpd (WordPerfect) -- md (Markdown) - xlsb (Excel binary workbook — an OOXML package whose workbook parts are binary rather than spreadsheetml) diff --git a/conan.lock b/conan.lock index 95b848def..05a598d74 100644 --- a/conan.lock +++ b/conan.lock @@ -11,6 +11,7 @@ "openjpeg/2.5.4#372fbc2b4348d45ab0c0a62a8475dc2f%1760446899.685", "nlohmann_json/3.12.0#2d634ab0ec8d9f56353e5ccef6d6612c%1744735883.94", "miniz/3.0.2#bfbce07c6654293cce27ee24129d2df7%1743673472.805", + "md4c/0.5.2#3d7106721e458f9f799b87d4d50d02e0%1746796758.933", "gtest/1.14.0#f8f0757a574a8dd747d16af62d6eb1b7%1743410807.169", "cryptopp/8.9.0#7a51e0038756b21bc3a6b82d681d5906%1758206597.119", "cpp-httplib/0.47.0#add6673ff352c26898ed2650453e706e%1784539639.401", diff --git a/conanfile.py b/conanfile.py index d3806476b..4afb5b35e 100644 --- a/conanfile.py +++ b/conanfile.py @@ -50,6 +50,7 @@ def config_options(self): def requirements(self): self.requires("pugixml/1.15") self.requires("cryptopp/8.9.0") + self.requires("md4c/0.5.2") self.requires("miniz/3.0.2") self.requires("nlohmann_json/3.12.0") self.requires("openjpeg/2.5.4") diff --git a/docs/design/README.md b/docs/design/README.md index bf0adb5a3..bd73e80cb 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -48,7 +48,6 @@ ### New format support -- markdown (candidates: [md4c](https://github.com/mity/md4c), [cmark](https://github.com/commonmark/cmark)) - xml - json - csv diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 61c6ecdd1..8fb113d73 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -452,14 +452,16 @@ constexpr std::array table{ .open = true, .translate_html = true, .color_scheme = true}}, - // Classified for callers that route files by type; there is no decoder. + // A document rather than a text file: it decodes to a `TextRoot`. Not + // `detect_by_content` — markdown has no signature, and a content probe for + // it is a probe for prose, so the caller routes on the file name. Row{FileType::markdown, "md"sv, markdown_extensions, markdown_mimetypes, - FileCategory::text, - DocumentType::unknown, - {}}, + FileCategory::document, + DocumentType::text, + {.open = true, .translate_html = true, .color_scheme = true}}, Row{FileType::zip, "zip"sv, diff --git a/src/odr/internal/markdown/AGENTS.md b/src/odr/internal/markdown/AGENTS.md new file mode 100644 index 000000000..8118b8f61 --- /dev/null +++ b/src/odr/internal/markdown/AGENTS.md @@ -0,0 +1,134 @@ +# AGENTS.md — `internal/markdown` + +Read the root [`AGENTS.md`](../../../../AGENTS.md) first, and +[`PLAN.md`](PLAN.md) for where this is going. This file covers what markdown +does differently, and why. + +## A document, not a text file + +`FileCategory::document` / `DocumentType::text`: a markdown file decodes to a +`TextRoot`, so the generic HTML renderer and every binding get it without +format-specific code. That is the whole argument for a decoder rather than a +markdown→HTML renderer next to `html/text_file.cpp` — the latter would produce +html only, with no element api and nothing for JNI/embind/pybind/ObjC. + +Unlike csv, markdown is *not* an `abstract::TextFile` that also loads as a +document. `abstract::TextFile` fixes `file_category()` to `text`, and there is +no reason to read markdown as a line list once it parses as prose. + +## Nothing detects it, and nothing rejects it + +`detect_by_content` is **false**. Markdown has no signature, and a content probe +for it is a probe for "prose with occasional punctuation" — every plain text +file with a `#` comment or an `*` bullet in it. Sniffing would steal `text_file` +matches and be confidently wrong. The only way in is +`DecodedFile(file, FileType::markdown)`; callers route on the file name, which +is what they already have. **A `.md` still opens as `text_file` by default**, +and that is correct for a format that is by construction valid plain text. + +There is no `NoMarkdownFile` either. Every other format's exception exists +because detection rejects; nothing rejects here — md4c is total, any UTF-8 byte +sequence is some markdown document. The only failure is an encoding +`internal/encoding` cannot decode, which throws `UnsupportedTextEncoding`: +`Text::content()` is UTF-8 to every binding, so legacy bytes have no document. + +## md4c, not a hand-rolled parser + +The opposite call from `d8c8715` (dropping a csv library to scan csv in-tree), +and deliberately so. Csv is a hundred lines of quote-aware scanning. CommonMark +has ~650 conformance cases, and the places hand-written parsers rot — lazy +continuation, link reference definitions, emphasis flanking rules, list-item +indent arithmetic — are exactly the ones that look easy for a weekend and are +then wrong forever. + +md4c is a SAX parser (`enter_block` / `leave_block` / `enter_span` / +`leave_span` / `text`), which maps onto `create_element` / `append_child` with +one id stack and no intermediate tree. Dialect: `MD_DIALECT_GITHUB` plus +`MD_FLAG_COLLAPSEWHITESPACE`. + +Two things the C boundary imposes: + +- **An exception must not unwind through md4c's frames.** Every callback runs + through `invoke`, which parks what it throws in the `Parser` and returns + non-zero; `parse_tree` rethrows once `md_parse` has returned. +- **md4c parses bytes and assumes UTF-8**, so decoding happens before it, in + `text::TextFile::text()`, not inside it. + +## Element mapping + +| md4c | model | +|---|---| +| `MD_BLOCK_DOC` | `root`, default (empty) `PageLayout` — markdown is flow content, not paged | +| `MD_BLOCK_H` | `paragraph` + heading `TextStyle`, plus a bold `span` (see below) | +| `MD_BLOCK_P` | `paragraph` | +| `MD_BLOCK_UL` / `OL` | `list`; `MD_BLOCK_LI` → `list_item` carrying its marker | +| `MD_BLOCK_QUOTE` | `group` + a left `margin` on the paragraphs inside, one step per level | +| `MD_BLOCK_CODE` | `group` of one monospace `paragraph` **per line** | +| `MD_BLOCK_HR` | — (nothing in the model) | +| `MD_BLOCK_HTML` | — (dropped) | +| `MD_BLOCK_TABLE` / `TR` / `TH` / `TD` | `table` / `table_row` / `table_cell`; `THEAD`/`TBODY` are transparent | +| `MD_SPAN_EM` / `STRONG` / `DEL` / `CODE` | `span` + the one style it means | +| `MD_SPAN_A` | `link` | +| `MD_SPAN_IMG` | — (transparent; the alt text flows through as text) | +| `MD_TEXT_BR` | `line_break`; `SOFTBR` is a space | +| `MD_TEXT_NULLCHAR` | U+FFFD | + +### Why a heading also gets a span + +`html::translate_paragraph` takes only **font family and size** from a +paragraph's text style (`translate_block_font_style`) — weight, slant and +decoration are expected on the spans inside. So a heading is a paragraph +carrying the whole heading style (that is where the level survives for the +element api) plus a span carrying `strong_style()` and nothing else. Only the +weight: the paragraph's size is already in `em`, and repeating it on the span +would compound. A `TH` cell is built the same way. + +### Why a code block is one paragraph per line + +The model has no pre-formatted block and `ParagraphStyle` has no +`white-space`, so a single paragraph would collapse the newlines. One paragraph +per line survives; `html::escape_text` turns leading and doubled spaces into +` `, so indentation survives with it. The info string (` ```cpp `) is +dropped — there is nowhere to put a language yet, which is stage 5 in +[`PLAN.md`](PLAN.md). + +### Why a tight list item opens a paragraph of its own + +md4c omits `MD_BLOCK_P` inside a *tight* list item, so its text arrives with no +block around it — and `html::translate_list_item` writes the item's marker +*into its first paragraph*, so without one the marker is silently dropped. +`open_implicit_paragraph_` opens one when inline content lands directly in a +`list_item`; the next block (or the item's `leave`) closes it again. + +### Why the columns hang off their own chain + +A table's children are its rows. One sibling chain cannot also carry the +columns, so `Table` keeps `first_column_id`/`last_column_id` and +`append_column` links them separately — the same shape `odf` uses. Getting +this wrong is not a compile error: the renderer walks `table_first_column`'s +siblings and happily writes a `` for every row it runs into. + +### Task lists + +The model has no checkbox, so the box (`☐` / `☑`) *replaces* the item's +marker rather than being invented as an element type. + +## Known gaps + +- **Named entities beyond the XML five (plus ` `) stay literal.** md4c + matches anything shaped like `&name;` without knowing the list, and the conan + package ships no `entity.h`. `entity_lookup` *is* linkable from the + `md4c-html` component, but its struct is private — using it means declaring + that layout ourselves, where a mismatch corrupts silently rather than failing + to link. Decide that before promising CommonMark conformance; vendoring the + table is the other option. +- **Raw html is dropped**, block and inline. There is no passthrough element, + and inventing one means deciding what `Text::content()` returns for it in four + bindings — a real question, not a markdown question. Inline `` therefore + renders as nothing. +- **A hyperlink's href is passed through** and only `escape_attribute`d by the + renderer, exactly as an odt's is. The root `AGENTS.md` flags that + inconsistency with `html/pdf_file.cpp`'s scheme allowlist; this module is a + second consumer of the document-link policy, not a third policy. +- **Horizontal rules, images and frontmatter** are not modelled yet — stages 4 + and 5 in [`PLAN.md`](PLAN.md). diff --git a/src/odr/internal/markdown/PLAN.md b/src/odr/internal/markdown/PLAN.md index 0de11438b..32a1c54ea 100644 --- a/src/odr/internal/markdown/PLAN.md +++ b/src/odr/internal/markdown/PLAN.md @@ -3,7 +3,11 @@ Where markdown support is going, and in what order. Written before stage 1; keep it honest as stages land. -## Today +**Stages 1 to 3 have landed.** What the module does and why is in +[`AGENTS.md`](AGENTS.md); what follows is the original plan, with the +after-the-fact corrections marked. Stages 4 and 5 are still open. + +## Today (before stage 1) `FileType::markdown` exists (`file.hpp:73`) and has a table row with extensions and mime types (`file_type_table.cpp:431`), declared classification-only: @@ -163,6 +167,14 @@ ordered list renders as bullets; and `ElementType::group` renders as its children with no wrapper (`:72`), so a blockquote's structure survives only in the margins its paragraphs carry. +> Corrected while landing stage 1. The list gap is gone — the renderer now +> writes `div`s with `role="list"` and takes the label from +> `list_item_marker`, so an ordered list numbers itself. The group gap stands. +> Three further corrections, all in [`AGENTS.md`](AGENTS.md): a heading also +> needs a bold `span` (a paragraph's text style contributes only family and +> size); a code block becomes one paragraph *per line*, not one paragraph; and +> a tight list item needs a paragraph opened for it or its marker is dropped. + ## Module layout Mirrors `oldms/text`, which is the reference the root `AGENTS.md` points at. @@ -180,7 +192,7 @@ Every `.cpp` goes into `ODR_SOURCE_FILES` (`CMakeLists.txt:86`). --- -## Stage 1 — blocks +## Stage 1 — blocks — **landed** The skeleton, end to end, with the least that is worth rendering. @@ -199,7 +211,10 @@ The skeleton, end to end, with the least that is worth rendering. engines do not exceed the row, so a handful of `.md` samples go into the test-data repo alongside this stage. -## Stage 2 — inlines and styles +`odr-public` already carries one `.md`, so that assertion has a file without +anything being added to the test-data repo. + +## Stage 2 — inlines and styles — **landed** - `SpanAdapter`, `LinkAdapter`; emphasis, strong, inline code, links. - `markdown_style`: one `StyleRegistry` handing out the heading scale, the @@ -208,7 +223,14 @@ test-data repo alongside this stage. - entity and soft-break handling per the table; the `entity_lookup` question above resolves here. -## Stage 3 — GFM +> The `entity_lookup` question resolved *against* it. The conan package does +> export the `md4c-html` component and the symbol is in the archive, but not +> `entity.h` — so using it means declaring md4c's private `MD_ENTITY` layout +> ourselves, which corrupts silently if it ever changes. Left out: numeric +> references, the five predefined XML entities and ` ` resolve, everything +> else stays literal. Revisit before claiming CommonMark conformance. + +## Stage 3 — GFM — **landed** `MD_DIALECT_GITHUB`: tables, strikethrough, task lists, permissive autolinks. Tables are the substantial one — `TableAdapter` plus row/column/cell — and the @@ -217,9 +239,19 @@ reason to do it before images: it is what people actually put in readmes. Task list items have no checkbox in the model. Render the box as text (`☐`/`☑`) in the item's first text element rather than inventing an element type for it. +> Landed as the item's *marker* rather than its first text element — that is +> what a marker is, and it keeps the box out of the item's reading text. + ## Stage 4 — images and frontmatter +Until this lands, `MD_SPAN_IMG` is transparent: the alt text flows through as +text, which is at least something to read. + - `FrameAdapter` + `ImageAdapter`, external hrefs per the decision above. + Watch the sizing: `html::translate_image` writes an `` at + `width:100%;height:100%` absolutely positioned inside the frame's `div`, so a + frame with no width and height renders nothing at all. Markdown carries no + dimensions, so that path needs an answer before a frame is worth creating. - YAML/TOML frontmatter: md4c does not know it, so strip a leading `---` fence before parsing and expose what it holds through `FileMeta`. Parse only the flat scalars we have somewhere to put (`title`, `author`, `date`); do not diff --git a/src/odr/internal/markdown/markdown_document.cpp b/src/odr/internal/markdown/markdown_document.cpp new file mode 100644 index 000000000..735ffc3d6 --- /dev/null +++ b/src/odr/internal/markdown/markdown_document.cpp @@ -0,0 +1,325 @@ +#include + +#include +#include +#include +#include + +#include +#include + +#include + +namespace odr::internal::markdown { + +namespace { +std::unique_ptr +create_element_adapter(const ElementRegistry ®istry, + const StyleRegistry &style_registry); +} + +Document::Document(const std::string_view text) + : internal::Document(FileType::markdown, DocumentType::text, nullptr) { + m_root_element = parse_tree(m_element_registry, m_style_registry, text); + + m_element_adapter = + create_element_adapter(m_element_registry, m_style_registry); +} + +const ElementRegistry &Document::element_registry() const { + return m_element_registry; +} + +const StyleRegistry &Document::style_registry() const { + return m_style_registry; +} + +bool Document::is_editable() const noexcept { return false; } + +bool Document::is_savable(const bool encrypted) const noexcept { + (void)encrypted; + return false; +} + +void Document::save(const Path &path) const { + (void)path; + throw UnsupportedOperation(); +} + +void Document::save(const Path &path, const char *password) const { + (void)path; + (void)password; + throw UnsupportedOperation(); +} + +namespace { + +class ElementAdapter final : public abstract::ElementAdapter, + public abstract::TextRootAdapter, + public abstract::LineBreakAdapter, + public abstract::ParagraphAdapter, + public abstract::SpanAdapter, + public abstract::TextAdapter, + public abstract::LinkAdapter, + public abstract::ListAdapter, + public abstract::ListItemAdapter, + public abstract::TableAdapter, + public abstract::TableColumnAdapter, + public abstract::TableRowAdapter, + public abstract::TableCellAdapter { +public: + ElementAdapter(const ElementRegistry ®istry, + const StyleRegistry &style_registry) + : m_registry(®istry), m_style_registry(&style_registry) {} + + [[nodiscard]] ElementType + element_type(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).type; + } + + [[nodiscard]] ElementIdentifier + element_parent(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).parent_id; + } + [[nodiscard]] ElementIdentifier + element_first_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).first_child_id; + } + [[nodiscard]] ElementIdentifier + element_last_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).last_child_id; + } + [[nodiscard]] ElementIdentifier + element_previous_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).previous_sibling_id; + } + [[nodiscard]] ElementIdentifier + element_next_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).next_sibling_id; + } + + [[nodiscard]] bool + element_is_unique(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_self_locatable(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_editable(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] DocumentPath + element_document_path(const ElementIdentifier element_id) const override { + return util::document::extract_path(*this, element_id, null_element_id); + } + [[nodiscard]] ElementIdentifier + element_navigate_path(const ElementIdentifier element_id, + const DocumentPath &path) const override { + return util::document::navigate_path(*this, element_id, path); + } + + [[nodiscard]] const TextRootAdapter * + text_root_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::root ? this : nullptr; + } + [[nodiscard]] const LineBreakAdapter * + line_break_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::line_break ? this : nullptr; + } + [[nodiscard]] const ParagraphAdapter * + paragraph_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::paragraph ? this : nullptr; + } + [[nodiscard]] const SpanAdapter * + span_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::span ? this : nullptr; + } + [[nodiscard]] const TextAdapter * + text_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::text ? this : nullptr; + } + [[nodiscard]] const LinkAdapter * + link_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::link ? this : nullptr; + } + [[nodiscard]] const ListAdapter * + list_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::list ? this : nullptr; + } + [[nodiscard]] const ListItemAdapter * + list_item_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::list_item ? this : nullptr; + } + [[nodiscard]] const TableAdapter * + table_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table ? this : nullptr; + } + [[nodiscard]] const TableColumnAdapter * + table_column_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_column ? this + : nullptr; + } + [[nodiscard]] const TableRowAdapter * + table_row_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_row ? this : nullptr; + } + [[nodiscard]] const TableCellAdapter * + table_cell_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_cell ? this : nullptr; + } + + /// Markdown is flow content: it has no page, and the viewport is the width. + [[nodiscard]] PageLayout + text_root_page_layout(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] ElementIdentifier text_root_first_master_page( + const ElementIdentifier element_id) const override { + (void)element_id; + return null_element_id; + } + + [[nodiscard]] TextStyle + line_break_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] ParagraphStyle + paragraph_style(const ElementIdentifier element_id) const override { + return m_style_registry->paragraph_style( + m_registry->element_paragraph_style_index(element_id)); + } + [[nodiscard]] TextStyle + paragraph_text_style(const ElementIdentifier element_id) const override { + return stored_text_style(element_id); + } + + [[nodiscard]] TextStyle + span_style(const ElementIdentifier element_id) const override { + return stored_text_style(element_id); + } + + [[nodiscard]] std::string + text_content(const ElementIdentifier element_id) const override { + return m_registry->text_element_at(element_id).text; + } + void text_set_content(const ElementIdentifier element_id, + const std::string &text) const override { + (void)element_id; + (void)text; + throw UnsupportedOperation(); + } + [[nodiscard]] TextStyle + text_style(const ElementIdentifier element_id) const override { + // The enclosing span or paragraph carries the character style. + (void)element_id; + return {}; + } + + [[nodiscard]] std::string + link_href(const ElementIdentifier element_id) const override { + return m_registry->link_element_at(element_id).href; + } + + [[nodiscard]] ListType + list_type(const ElementIdentifier element_id) const override { + return m_registry->list_element_at(element_id).type; + } + + [[nodiscard]] TextStyle + list_item_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] std::string + list_item_marker(const ElementIdentifier element_id) const override { + return m_registry->list_item_element_at(element_id).marker; + } + [[nodiscard]] std::optional + list_item_number(const ElementIdentifier element_id) const override { + return m_registry->list_item_element_at(element_id).number; + } + + [[nodiscard]] TableDimensions + table_dimensions(const ElementIdentifier element_id) const override { + return m_registry->table_element_at(element_id).dimensions; + } + [[nodiscard]] ElementIdentifier + table_first_column(const ElementIdentifier element_id) const override { + return m_registry->table_element_at(element_id).first_column_id; + } + /// The rows are the table's children; the columns hang off a chain of their + /// own, as they do in odf. + [[nodiscard]] ElementIdentifier + table_first_row(const ElementIdentifier element_id) const override { + return element_first_child(element_id); + } + [[nodiscard]] TableStyle + table_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TableColumnStyle + table_column_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TableRowStyle + table_row_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + /// A markdown table is a grid of single cells: no merging, no formulas. + [[nodiscard]] bool + table_cell_is_covered(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] TableDimensions + table_cell_span(const ElementIdentifier element_id) const override { + (void)element_id; + return {1, 1}; + } + [[nodiscard]] ValueType + table_cell_value_type(const ElementIdentifier element_id) const override { + (void)element_id; + return ValueType::string; + } + [[nodiscard]] TableCellStyle + table_cell_style(const ElementIdentifier element_id) const override { + TableCellStyle result; + result.horizontal_align = + m_registry->table_cell_element_at(element_id).horizontal_align; + return result; + } + +private: + const ElementRegistry *m_registry{nullptr}; + const StyleRegistry *m_style_registry{nullptr}; + + [[nodiscard]] TextStyle + stored_text_style(const ElementIdentifier element_id) const { + return m_style_registry->text_style( + m_registry->element_text_style_index(element_id)); + } +}; + +std::unique_ptr +create_element_adapter(const ElementRegistry ®istry, + const StyleRegistry &style_registry) { + return std::make_unique(registry, style_registry); +} + +} // namespace + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_document.hpp b/src/odr/internal/markdown/markdown_document.hpp new file mode 100644 index 000000000..10aaa71fe --- /dev/null +++ b/src/odr/internal/markdown/markdown_document.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include + +namespace odr::internal::markdown { + +/// A markdown file as a text document: the whole tree is built up front from +/// the decoded UTF-8 text, and the file is not needed again. +class Document final : public internal::Document { +public: + explicit Document(std::string_view text); + + [[nodiscard]] const ElementRegistry &element_registry() const; + + [[nodiscard]] const StyleRegistry &style_registry() const; + + [[nodiscard]] bool is_editable() const noexcept override; + [[nodiscard]] bool is_savable(bool encrypted) const noexcept override; + + void save(const Path &path) const override; + void save(const Path &path, const char *password) const override; + +private: + ElementRegistry m_element_registry; + StyleRegistry m_style_registry; +}; + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_element_registry.cpp b/src/odr/internal/markdown/markdown_element_registry.cpp new file mode 100644 index 000000000..f5dc63496 --- /dev/null +++ b/src/odr/internal/markdown/markdown_element_registry.cpp @@ -0,0 +1,208 @@ +#include + +#include + +namespace odr::internal::markdown { + +namespace { + +/// Looks a payload up, turning a missing entry into an out-of-range error +/// rather than the map's default-constructed one. +template auto &payload_at(Map &map, const ElementIdentifier id) { + const auto it = map.find(id); + if (it == map.end()) { + throw std::out_of_range( + "markdown::ElementRegistry: element has no payload of that kind"); + } + return it->second; +} + +} // namespace + +std::size_t ElementRegistry::size() const noexcept { return m_elements.size(); } + +std::tuple +ElementRegistry::create_element(const ElementType type) { + Element &element = m_elements.emplace_back(); + const ElementIdentifier element_id = m_elements.size(); + element.type = type; + return {element_id, element}; +} + +std::tuple +ElementRegistry::create_text_element() { + const auto &[element_id, element] = create_element(ElementType::text); + const auto [it, inserted] = m_texts.emplace(element_id, Text{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_link_element() { + const auto &[element_id, element] = create_element(ElementType::link); + const auto [it, inserted] = m_links.emplace(element_id, Link{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_list_element() { + const auto &[element_id, element] = create_element(ElementType::list); + const auto [it, inserted] = m_lists.emplace(element_id, List{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_list_item_element() { + const auto &[element_id, element] = create_element(ElementType::list_item); + const auto [it, inserted] = m_list_items.emplace(element_id, ListItem{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_table_element() { + const auto &[element_id, element] = create_element(ElementType::table); + const auto [it, inserted] = m_tables.emplace(element_id, Table{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_table_cell_element() { + const auto &[element_id, element] = create_element(ElementType::table_cell); + const auto [it, inserted] = m_table_cells.emplace(element_id, TableCell{}); + return {element_id, element, it->second}; +} + +ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) { + check_element_id(id); + return m_elements.at(id - 1); +} + +ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) { + return payload_at(m_texts, id); +} + +ElementRegistry::Table & +ElementRegistry::table_element_at(const ElementIdentifier id) { + return payload_at(m_tables, id); +} + +const ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) const { + check_element_id(id); + return m_elements.at(id - 1); +} + +const ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) const { + return payload_at(m_texts, id); +} + +const ElementRegistry::Link & +ElementRegistry::link_element_at(const ElementIdentifier id) const { + return payload_at(m_links, id); +} + +const ElementRegistry::List & +ElementRegistry::list_element_at(const ElementIdentifier id) const { + return payload_at(m_lists, id); +} + +const ElementRegistry::ListItem & +ElementRegistry::list_item_element_at(const ElementIdentifier id) const { + return payload_at(m_list_items, id); +} + +const ElementRegistry::Table & +ElementRegistry::table_element_at(const ElementIdentifier id) const { + return payload_at(m_tables, id); +} + +const ElementRegistry::TableCell & +ElementRegistry::table_cell_element_at(const ElementIdentifier id) const { + return payload_at(m_table_cells, id); +} + +void ElementRegistry::append_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id) { + check_element_id(parent_id); + check_element_id(child_id); + if (element_at(child_id).parent_id != null_element_id) { + throw std::invalid_argument( + "markdown::ElementRegistry::append_child: child already has a parent"); + } + + Element &parent = element_at(parent_id); + link_child(parent_id, child_id, parent.first_child_id, parent.last_child_id); +} + +void ElementRegistry::append_column(const ElementIdentifier table_id, + const ElementIdentifier column_id) { + check_element_id(column_id); + if (element_at(column_id).parent_id != null_element_id) { + throw std::invalid_argument( + "markdown::ElementRegistry::append_column: child already has a parent"); + } + + Table &table = table_element_at(table_id); + link_child(table_id, column_id, table.first_column_id, table.last_column_id); +} + +void ElementRegistry::link_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id, + ElementIdentifier &first_id, + ElementIdentifier &last_id) { + Element &child = element_at(child_id); + child.parent_id = parent_id; + child.previous_sibling_id = last_id; + + if (first_id == null_element_id) { + first_id = child_id; + } else { + element_at(last_id).next_sibling_id = child_id; + } + last_id = child_id; +} + +void ElementRegistry::set_element_text_style_index(const ElementIdentifier id, + const std::uint32_t index) { + check_element_id(id); + m_text_style_indices[id] = index; +} + +std::uint32_t +ElementRegistry::element_text_style_index(const ElementIdentifier id) const { + const auto it = m_text_style_indices.find(id); + return it != m_text_style_indices.end() ? it->second : 0; +} + +void ElementRegistry::set_element_paragraph_style_index( + const ElementIdentifier id, const std::uint32_t index) { + check_element_id(id); + m_paragraph_style_indices[id] = index; +} + +std::uint32_t ElementRegistry::element_paragraph_style_index( + const ElementIdentifier id) const { + const auto it = m_paragraph_style_indices.find(id); + return it != m_paragraph_style_indices.end() ? it->second : 0; +} + +void ElementRegistry::check_element_id(const ElementIdentifier id) const { + if (id == null_element_id) { + throw std::out_of_range( + "markdown::ElementRegistry::check_element_id: null identifier"); + } + if (id - 1 >= m_elements.size()) { + throw std::out_of_range( + "markdown::ElementRegistry::check_element_id: identifier out of range"); + } +} + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_element_registry.hpp b/src/odr/internal/markdown/markdown_element_registry.hpp new file mode 100644 index 000000000..05ec095b6 --- /dev/null +++ b/src/odr/internal/markdown/markdown_element_registry.hpp @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace odr::internal::markdown { + +class ElementRegistry final { +public: + struct Element final { + ElementIdentifier parent_id{null_element_id}; + ElementIdentifier first_child_id{null_element_id}; + ElementIdentifier last_child_id{null_element_id}; + ElementIdentifier previous_sibling_id{null_element_id}; + ElementIdentifier next_sibling_id{null_element_id}; + ElementType type{ElementType::none}; + }; + + struct Text final { + std::string text; + }; + + struct Link final { + std::string href; + }; + + struct List final { + ListType type{ListType::unordered}; + }; + + struct ListItem final { + std::string marker; + std::optional number; + }; + + /// The columns hang off the table on a chain of their own, as they do in + /// odf: the rows are the table's children, and one sibling chain cannot + /// carry both. + struct Table final { + TableDimensions dimensions; + ElementIdentifier first_column_id{null_element_id}; + ElementIdentifier last_column_id{null_element_id}; + }; + + struct TableCell final { + std::optional horizontal_align; + }; + + [[nodiscard]] std::size_t size() const noexcept; + + std::tuple create_element(ElementType type); + std::tuple create_text_element(); + std::tuple create_link_element(); + std::tuple create_list_element(); + std::tuple + create_list_item_element(); + std::tuple create_table_element(); + std::tuple + create_table_cell_element(); + + [[nodiscard]] Element &element_at(ElementIdentifier id); + [[nodiscard]] Text &text_element_at(ElementIdentifier id); + [[nodiscard]] Table &table_element_at(ElementIdentifier id); + + [[nodiscard]] const Element &element_at(ElementIdentifier id) const; + [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; + [[nodiscard]] const Link &link_element_at(ElementIdentifier id) const; + [[nodiscard]] const List &list_element_at(ElementIdentifier id) const; + [[nodiscard]] const ListItem & + list_item_element_at(ElementIdentifier id) const; + [[nodiscard]] const Table &table_element_at(ElementIdentifier id) const; + [[nodiscard]] const TableCell & + table_cell_element_at(ElementIdentifier id) const; + + void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + void append_column(ElementIdentifier table_id, ElementIdentifier column_id); + + /// Character style of an element, as an index into the document's + /// `StyleRegistry` (0 is the default style). + void set_element_text_style_index(ElementIdentifier id, std::uint32_t index); + [[nodiscard]] std::uint32_t + element_text_style_index(ElementIdentifier id) const; + + /// Paragraph style of an element, indexed the same way. + void set_element_paragraph_style_index(ElementIdentifier id, + std::uint32_t index); + [[nodiscard]] std::uint32_t + element_paragraph_style_index(ElementIdentifier id) const; + +private: + std::vector m_elements; + std::unordered_map m_texts; + std::unordered_map m_links; + std::unordered_map m_lists; + std::unordered_map m_list_items; + std::unordered_map m_tables; + std::unordered_map m_table_cells; + std::unordered_map m_text_style_indices; + std::unordered_map + m_paragraph_style_indices; + + void check_element_id(ElementIdentifier id) const; + + /// Links @p child_id onto the chain @p first_id / @p last_id delimit. + void link_child(ElementIdentifier parent_id, ElementIdentifier child_id, + ElementIdentifier &first_id, ElementIdentifier &last_id); +}; + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_file.cpp b/src/odr/internal/markdown/markdown_file.cpp new file mode 100644 index 000000000..927e4ab76 --- /dev/null +++ b/src/odr/internal/markdown/markdown_file.cpp @@ -0,0 +1,54 @@ +#include + +#include +#include + +#include + +#include +#include + +namespace odr::internal::markdown { + +MarkdownFile::MarkdownFile(std::shared_ptr file) + : m_file{std::move(file)} {} + +std::shared_ptr MarkdownFile::file() const noexcept { + return m_file->file(); +} + +FileType MarkdownFile::file_type() const noexcept { return FileType::markdown; } + +std::string_view MarkdownFile::mimetype() const noexcept { + return "text/markdown"; +} + +FileMeta MarkdownFile::file_meta() const noexcept { + FileMeta result; + result.type = file_type(); + result.mimetype = mimetype(); + result.document_type = DocumentType::text; + return result; +} + +DocumentType MarkdownFile::document_type() const { return DocumentType::text; } + +bool MarkdownFile::is_decodable() const noexcept { + return text_encoding_is_decodable(encoding()); +} + +std::shared_ptr MarkdownFile::document() const { + // `Text::content()` is UTF-8 to every binding, so bytes we cannot decode + // have no document at all — the text rendering path stays open to them. + if (!is_decodable()) { + throw UnsupportedTextEncoding(encoding()); + } + const std::string text = m_file->text(); + return std::make_shared(text); +} + +TextEncoding MarkdownFile::encoding() const noexcept { + return m_file->encoding(); +} + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_file.hpp b/src/odr/internal/markdown/markdown_file.hpp new file mode 100644 index 000000000..31ee0ed78 --- /dev/null +++ b/src/odr/internal/markdown/markdown_file.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace odr::internal::markdown { + +/// Markdown is a document, not a text file: it decodes to a `TextRoot`, so +/// `FileCategory::document` and `DocumentType::text`. Nothing here rejects — +/// any UTF-8 byte sequence is some markdown — so there is no `NoMarkdownFile`; +/// only an undecodable encoding stops it. See `AGENTS.md`. +class MarkdownFile final : public abstract::DocumentFile { +public: + explicit MarkdownFile(std::shared_ptr file); + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] DocumentType document_type() const override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + /// @throws UnsupportedTextEncoding if the encoding cannot be decoded. + [[nodiscard]] std::shared_ptr document() const override; + + /// The encoding the bytes were detected as. + [[nodiscard]] TextEncoding encoding() const noexcept; + +private: + std::shared_ptr m_file; +}; + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_parser.cpp b/src/odr/internal/markdown/markdown_parser.cpp new file mode 100644 index 000000000..bddc6e695 --- /dev/null +++ b/src/odr/internal/markdown/markdown_parser.cpp @@ -0,0 +1,569 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal::markdown { + +namespace { + +/// U+FFFD, the replacement CommonMark prescribes for a NULL character. +constexpr std::string_view replacement_character = "�"; + +/// Resolves a numeric character reference, or the five predefined XML entities +/// plus ` `. md4c matches anything shaped like a named entity without +/// knowing the list and the package exports no lookup table, so everything +/// else stays literal — which is what a reader of the source sees too. +std::string resolve_entity(const std::string_view entity) { + if (entity.size() < 3 || entity.front() != '&' || entity.back() != ';') { + return std::string(entity); + } + const std::string_view name = entity.substr(1, entity.size() - 2); + + if (name.front() == '#') { + const bool hexadecimal = + name.size() > 1 && (name[1] == 'x' || name[1] == 'X'); + const std::string_view digits = name.substr(hexadecimal ? 2 : 1); + std::uint32_t code_point{0}; + const auto [end, error] = + std::from_chars(digits.data(), digits.data() + digits.size(), + code_point, hexadecimal ? 16 : 10); + if (error != std::errc() || end != digits.data() + digits.size()) { + return std::string(entity); + } + // an unpaired surrogate or an out-of-range value is not a character + if (code_point == 0 || code_point > 0x10FFFF || + (code_point >= 0xD800 && code_point <= 0xDFFF)) { + return std::string(replacement_character); + } + std::string result; + util::string::append_c32(static_cast(code_point), result); + return result; + } + + if (name == "amp") { + return "&"; + } + if (name == "lt") { + return "<"; + } + if (name == "gt") { + return ">"; + } + if (name == "quot") { + return "\""; + } + if (name == "apos") { + return "'"; + } + if (name == "nbsp") { + return " "; + } + return std::string(entity); +} + +/// The text of an `MD_ATTRIBUTE` — a link href or an image source — with its +/// entity substrings resolved. +std::string attribute_to_string(const MD_ATTRIBUTE &attribute) { + std::string result; + if (attribute.text == nullptr) { + return result; + } + for (std::size_t i = 0; attribute.substr_offsets[i] < attribute.size; ++i) { + const MD_OFFSET begin = attribute.substr_offsets[i]; + const MD_OFFSET end = attribute.substr_offsets[i + 1]; + const std::string_view part(attribute.text + begin, end - begin); + + switch (attribute.substr_types[i]) { + case MD_TEXT_ENTITY: + result.append(resolve_entity(part)); + break; + case MD_TEXT_NULLCHAR: + result.append(replacement_character); + break; + default: + result.append(part); + break; + } + } + return result; +} + +std::optional horizontal_align(const MD_ALIGN align) { + switch (align) { + case MD_ALIGN_LEFT: + return HorizontalAlign::left; + case MD_ALIGN_CENTER: + return HorizontalAlign::center; + case MD_ALIGN_RIGHT: + return HorizontalAlign::right; + case MD_ALIGN_DEFAULT: + default: + return {}; + } +} + +/// md4c's callbacks against one element stack: every block and span that has an +/// element pushes it, and its `leave` pops it again. See `AGENTS.md` for what +/// is deliberately mapped to nothing. +class Parser final { +public: + Parser(ElementRegistry ®istry, StyleRegistry &style_registry) + : m_registry{®istry}, m_style_registry{&style_registry} {} + + [[nodiscard]] ElementIdentifier root() const noexcept { return m_root; } + + void enter_block(MD_BLOCKTYPE type, const void *detail); + void leave_block(MD_BLOCKTYPE type); + void enter_span(MD_SPANTYPE type, const void *detail); + void leave_span(MD_SPANTYPE type); + void text(MD_TEXTTYPE type, std::string_view text); + + /// A callback cannot throw through md4c's C frames, so the exception is + /// parked here and rethrown once `md_parse` has returned. + void fail(const std::exception_ptr &error) noexcept { m_error = error; } + void rethrow() const { + if (m_error) { + std::rethrow_exception(m_error); + } + } + +private: + struct ListState final { + ListType type{ListType::unordered}; + std::uint32_t next_number{1}; + char delimiter{'.'}; + }; + + ElementRegistry *m_registry{nullptr}; + StyleRegistry *m_style_registry{nullptr}; + + ElementIdentifier m_root{null_element_id}; + std::vector m_stack; + /// The text element further text is appended to, while one is open. + ElementIdentifier m_open_text{null_element_id}; + + std::vector m_lists; + /// How many elements each open block pushed, so its `leave` pops as many. + std::vector m_block_pushes; + /// A tight list item holds its text without an `MD_BLOCK_P` around it; the + /// renderer writes the item's marker into its first paragraph, so one is + /// opened for the inline content and closed by the next block. + ElementIdentifier m_implicit_paragraph{null_element_id}; + + std::uint32_t m_quote_depth{0}; + /// Set between `MD_BLOCK_CODE`'s enter and leave; its text is buffered + /// because a code block becomes one paragraph per line. + bool m_in_code_block{false}; + std::string m_code; + + std::exception_ptr m_error; + + [[nodiscard]] ElementIdentifier current_() const; + void push_(ElementIdentifier element_id); + void pop_(); + + void push_paragraph_(std::uint32_t text_style_index); + void push_span_(std::uint32_t text_style_index); + void open_implicit_paragraph_(); + void close_implicit_paragraph_(); + void append_text_(std::string_view text); + void flush_code_(); +}; + +ElementIdentifier Parser::current_() const { + if (m_stack.empty()) { + throw std::runtime_error("markdown: content outside the document block"); + } + return m_stack.back(); +} + +void Parser::push_(const ElementIdentifier element_id) { + if (!m_stack.empty()) { + m_registry->append_child(m_stack.back(), element_id); + } + m_stack.push_back(element_id); + m_open_text = null_element_id; +} + +void Parser::pop_() { + if (m_stack.empty()) { + throw std::runtime_error("markdown: unbalanced md4c callbacks"); + } + m_stack.pop_back(); + m_open_text = null_element_id; +} + +void Parser::push_paragraph_(const std::uint32_t text_style_index) { + const auto &[element_id, element] = + m_registry->create_element(ElementType::paragraph); + if (text_style_index != 0) { + m_registry->set_element_text_style_index(element_id, text_style_index); + } + if (const std::uint32_t paragraph_style_index = + m_style_registry->quote_style(m_quote_depth); + paragraph_style_index != 0) { + m_registry->set_element_paragraph_style_index(element_id, + paragraph_style_index); + } + push_(element_id); +} + +void Parser::push_span_(const std::uint32_t text_style_index) { + open_implicit_paragraph_(); + const auto &[element_id, element] = + m_registry->create_element(ElementType::span); + m_registry->set_element_text_style_index(element_id, text_style_index); + push_(element_id); +} + +void Parser::open_implicit_paragraph_() { + if (m_registry->element_at(current_()).type != ElementType::list_item) { + return; + } + push_paragraph_(0); + m_implicit_paragraph = m_stack.back(); +} + +void Parser::close_implicit_paragraph_() { + if (m_implicit_paragraph == null_element_id || m_stack.empty() || + m_stack.back() != m_implicit_paragraph) { + return; + } + pop_(); + m_implicit_paragraph = null_element_id; +} + +void Parser::append_text_(const std::string_view text) { + if (text.empty()) { + return; + } + open_implicit_paragraph_(); + if (m_open_text == null_element_id) { + auto [element_id, element, payload] = m_registry->create_text_element(); + m_registry->append_child(current_(), element_id); + m_open_text = element_id; + payload.text.append(text); + return; + } + m_registry->text_element_at(m_open_text).text.append(text); +} + +void Parser::flush_code_() { + std::string_view remainder = m_code; + // md4c terminates every code line with '\n', the last one included + if (remainder.ends_with('\n')) { + remainder.remove_suffix(1); + } + + while (true) { + const std::size_t line_end = remainder.find('\n'); + const std::string_view line = remainder.substr(0, line_end); + + push_paragraph_(m_style_registry->monospace_style()); + append_text_(line); + pop_(); + + if (line_end == std::string_view::npos) { + break; + } + remainder.remove_prefix(line_end + 1); + } + + m_code.clear(); +} + +void Parser::enter_block(const MD_BLOCKTYPE type, const void *detail) { + close_implicit_paragraph_(); + const std::size_t depth = m_stack.size(); + + switch (type) { + case MD_BLOCK_DOC: { + const auto &[element_id, element] = + m_registry->create_element(ElementType::root); + m_root = element_id; + push_(element_id); + } break; + case MD_BLOCK_QUOTE: { + const auto &[element_id, element] = + m_registry->create_element(ElementType::group); + push_(element_id); + ++m_quote_depth; + } break; + case MD_BLOCK_UL: { + auto [element_id, element, list] = m_registry->create_list_element(); + list.type = ListType::unordered; + push_(element_id); + m_lists.push_back({}); + } break; + case MD_BLOCK_OL: { + const auto *ol = static_cast(detail); + auto [element_id, element, list] = m_registry->create_list_element(); + list.type = ListType::ordered; + push_(element_id); + m_lists.push_back( + {ListType::ordered, ol->start, static_cast(ol->mark_delimiter)}); + } break; + case MD_BLOCK_LI: { + const auto *li = static_cast(detail); + if (m_lists.empty()) { + throw std::runtime_error("markdown: list item outside a list"); + } + ListState &list = m_lists.back(); + + auto [element_id, element, item] = m_registry->create_list_item_element(); + if (li->is_task != 0) { + // The model has no checkbox, so the box replaces the item's marker. + item.marker = li->task_mark == 'x' || li->task_mark == 'X' ? "☑" : "☐"; + } else if (list.type == ListType::ordered) { + item.number = list.next_number; + item.marker = std::to_string(list.next_number) + list.delimiter; + } else { + item.marker = "•"; + } + if (list.type == ListType::ordered) { + ++list.next_number; + } + push_(element_id); + } break; + case MD_BLOCK_HR: + break; // nothing in the model + case MD_BLOCK_H: { + const auto *heading = static_cast(detail); + push_paragraph_(m_style_registry->heading_style(heading->level)); + // The renderer takes only font family and size from a paragraph's text + // style, so the weight has to ride on a span — and only the weight: the + // heading's `em` size is already the span's font size to be relative to. + push_span_(m_style_registry->strong_style()); + } break; + case MD_BLOCK_CODE: { + const auto &[element_id, element] = + m_registry->create_element(ElementType::group); + push_(element_id); + m_in_code_block = true; + m_code.clear(); + } break; + case MD_BLOCK_HTML: + break; // dropped, see `AGENTS.md` + case MD_BLOCK_P: + push_paragraph_(0); + break; + case MD_BLOCK_TABLE: { + const auto *table_detail = + static_cast(detail); + auto [element_id, element, table] = m_registry->create_table_element(); + table.dimensions = TableDimensions(table_detail->head_row_count + + table_detail->body_row_count, + table_detail->col_count); + push_(element_id); + + for (unsigned i = 0; i < table_detail->col_count; ++i) { + const auto &[column_id, column] = + m_registry->create_element(ElementType::table_column); + m_registry->append_column(element_id, column_id); + } + } break; + case MD_BLOCK_THEAD: + case MD_BLOCK_TBODY: + break; // the rows hang off the table itself + case MD_BLOCK_TR: { + const auto &[element_id, element] = + m_registry->create_element(ElementType::table_row); + push_(element_id); + } break; + case MD_BLOCK_TH: + case MD_BLOCK_TD: { + const auto *cell_detail = static_cast(detail); + auto [element_id, element, cell] = m_registry->create_table_cell_element(); + cell.horizontal_align = horizontal_align(cell_detail->align); + push_(element_id); + // A cell holds blocks; md4c hands its content over as inline text. + push_paragraph_(0); + if (type == MD_BLOCK_TH) { + push_span_(m_style_registry->strong_style()); + } + } break; + } + + m_block_pushes.push_back(m_stack.size() - depth); +} + +void Parser::leave_block(const MD_BLOCKTYPE type) { + close_implicit_paragraph_(); + + switch (type) { + case MD_BLOCK_QUOTE: + --m_quote_depth; + break; + case MD_BLOCK_UL: + case MD_BLOCK_OL: + m_lists.pop_back(); + break; + case MD_BLOCK_CODE: + m_in_code_block = false; + flush_code_(); + break; + default: + break; + } + + if (m_block_pushes.empty()) { + throw std::runtime_error("markdown: unbalanced md4c callbacks"); + } + for (std::size_t i = 0; i < m_block_pushes.back(); ++i) { + pop_(); + } + m_block_pushes.pop_back(); +} + +void Parser::enter_span(const MD_SPANTYPE type, const void *detail) { + switch (type) { + case MD_SPAN_EM: + push_span_(m_style_registry->emphasis_style()); + break; + case MD_SPAN_STRONG: + push_span_(m_style_registry->strong_style()); + break; + case MD_SPAN_DEL: + push_span_(m_style_registry->strikethrough_style()); + break; + case MD_SPAN_CODE: + push_span_(m_style_registry->monospace_style()); + break; + case MD_SPAN_A: { + const auto *link_detail = static_cast(detail); + open_implicit_paragraph_(); + auto [element_id, element, link] = m_registry->create_link_element(); + link.href = attribute_to_string(link_detail->href); + push_(element_id); + } break; + default: + break; // an image's alt text passes through as text, see `AGENTS.md` + } +} + +void Parser::leave_span(const MD_SPANTYPE type) { + switch (type) { + case MD_SPAN_EM: + case MD_SPAN_STRONG: + case MD_SPAN_DEL: + case MD_SPAN_CODE: + case MD_SPAN_A: + pop_(); + break; + default: + break; + } +} + +void Parser::text(const MD_TEXTTYPE type, const std::string_view text) { + switch (type) { + case MD_TEXT_NULLCHAR: + append_text_(replacement_character); + break; + case MD_TEXT_BR: { + open_implicit_paragraph_(); + const auto &[element_id, element] = + m_registry->create_element(ElementType::line_break); + m_registry->append_child(current_(), element_id); + m_open_text = null_element_id; + } break; + case MD_TEXT_SOFTBR: + append_text_(" "); + break; + case MD_TEXT_ENTITY: + append_text_(resolve_entity(text)); + break; + case MD_TEXT_CODE: + if (m_in_code_block) { + m_code.append(text); + } else { + append_text_(text); + } + break; + case MD_TEXT_HTML: + break; // dropped, see `AGENTS.md` + default: + append_text_(text); + break; + } +} + +/// Runs one callback body, parking anything it throws: an exception must not +/// unwind through md4c's C frames. +template int invoke(void *userdata, Callback &&callback) { + auto *parser = static_cast(userdata); + try { + callback(*parser); + } catch (...) { + parser->fail(std::current_exception()); + return 1; + } + return 0; +} + +} // namespace +} // namespace odr::internal::markdown + +namespace odr::internal { + +ElementIdentifier markdown::parse_tree(ElementRegistry ®istry, + StyleRegistry &style_registry, + const std::string_view text) { + Parser parser(registry, style_registry); + + MD_PARSER md_parser{}; + md_parser.flags = MD_DIALECT_GITHUB | MD_FLAG_COLLAPSEWHITESPACE; + md_parser.enter_block = [](const MD_BLOCKTYPE type, void *detail, + void *userdata) { + return invoke(userdata, + [&](Parser &parser) { parser.enter_block(type, detail); }); + }; + md_parser.leave_block = [](const MD_BLOCKTYPE type, void *, void *userdata) { + return invoke(userdata, [&](Parser &parser) { parser.leave_block(type); }); + }; + md_parser.enter_span = [](const MD_SPANTYPE type, void *detail, + void *userdata) { + return invoke(userdata, + [&](Parser &parser) { parser.enter_span(type, detail); }); + }; + md_parser.leave_span = [](const MD_SPANTYPE type, void *, void *userdata) { + return invoke(userdata, [&](Parser &parser) { parser.leave_span(type); }); + }; + md_parser.text = [](const MD_TEXTTYPE type, const MD_CHAR *text, + const MD_SIZE size, void *userdata) { + return invoke(userdata, [&](Parser &parser) { + parser.text(type, std::string_view(text, size)); + }); + }; + + // `md_parse` dereferences the pointer even for an empty document + const int result = + md_parse(text.empty() ? "" : text.data(), + static_cast(text.size()), &md_parser, &parser); + parser.rethrow(); + if (result != 0) { + throw std::runtime_error("markdown: md4c failed to parse the document"); + } + + return parser.root(); +} + +} // namespace odr::internal diff --git a/src/odr/internal/markdown/markdown_parser.hpp b/src/odr/internal/markdown/markdown_parser.hpp new file mode 100644 index 000000000..21f2c13d2 --- /dev/null +++ b/src/odr/internal/markdown/markdown_parser.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace odr::internal::markdown { +class ElementRegistry; +class StyleRegistry; + +/// Parses @p text — UTF-8 CommonMark plus the GitHub extensions — into +/// @p registry and @p style_registry, and returns the root element. +/// @throws std::runtime_error if md4c fails. +ElementIdentifier parse_tree(ElementRegistry ®istry, + StyleRegistry &style_registry, + std::string_view text); + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_style.cpp b/src/odr/internal/markdown/markdown_style.cpp new file mode 100644 index 000000000..e38469ff6 --- /dev/null +++ b/src/odr/internal/markdown/markdown_style.cpp @@ -0,0 +1,91 @@ +#include + +#include +#include +#include + +namespace odr::internal::markdown { + +namespace { + +/// The browsers' default heading scale, in `em` so it composes with whatever +/// font size the viewer sets. +constexpr std::array heading_font_sizes{2.0, 1.5, 1.17, 1.0, 0.83, 0.67}; + +/// One `
` worth of indent, per nesting level. +constexpr double quote_margin = 2.5; + +/// A generic family rather than a face: nothing in a markdown file names one, +/// and the viewer's monospace font is the closest thing to an author's intent. +/// Static storage, so `TextStyle::font_name` may point at it. +constexpr std::string_view monospace_font_name = "monospace"; + +constexpr std::uint32_t default_style_index = 0; +constexpr std::uint32_t first_heading_style_index = 1; +constexpr std::uint32_t monospace_style_index = 7; +constexpr std::uint32_t emphasis_style_index = 8; +constexpr std::uint32_t strong_style_index = 9; +constexpr std::uint32_t strikethrough_style_index = 10; + +} // namespace + +StyleRegistry::StyleRegistry() { + m_text_styles.resize(strikethrough_style_index + 1); + + for (std::size_t i = 0; i < heading_font_sizes.size(); ++i) { + TextStyle &style = m_text_styles.at(first_heading_style_index + i); + style.font_size = Measure(heading_font_sizes.at(i), DynamicUnit("em")); + style.font_weight = FontWeight::bold; + } + + m_text_styles.at(monospace_style_index).font_name = monospace_font_name; + m_text_styles.at(emphasis_style_index).font_style = FontStyle::italic; + m_text_styles.at(strong_style_index).font_weight = FontWeight::bold; + m_text_styles.at(strikethrough_style_index).font_line_through = true; + + m_paragraph_styles.resize(default_style_index + 1); +} + +const TextStyle &StyleRegistry::text_style(const std::uint32_t index) const { + return m_text_styles.at(index); +} + +const ParagraphStyle & +StyleRegistry::paragraph_style(const std::uint32_t index) const { + return m_paragraph_styles.at(index); +} + +std::uint32_t StyleRegistry::heading_style(const std::uint32_t level) const { + if (level < 1 || level > heading_font_sizes.size()) { + throw std::out_of_range("markdown: heading level out of range"); + } + return first_heading_style_index + level - 1; +} + +std::uint32_t StyleRegistry::monospace_style() const { + return monospace_style_index; +} + +std::uint32_t StyleRegistry::emphasis_style() const { + return emphasis_style_index; +} + +std::uint32_t StyleRegistry::strong_style() const { return strong_style_index; } + +std::uint32_t StyleRegistry::strikethrough_style() const { + return strikethrough_style_index; +} + +std::uint32_t StyleRegistry::quote_style(const std::uint32_t depth) { + if (depth == 0) { + return default_style_index; + } + while (m_paragraph_styles.size() <= depth) { + ParagraphStyle &style = m_paragraph_styles.emplace_back(); + style.margin.left = Measure(quote_margin * (m_paragraph_styles.size() - 1), + DynamicUnit("em")); + } + return depth; +} + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/markdown/markdown_style.hpp b/src/odr/internal/markdown/markdown_style.hpp new file mode 100644 index 000000000..1a98c8da3 --- /dev/null +++ b/src/odr/internal/markdown/markdown_style.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include + +namespace odr::internal::markdown { + +/// Owns the document's styles, indexed by the style index stored on elements; +/// 0 is the default style in both sets. Markdown carries no style information +/// of its own, so the set is a fixed rendering convention: a heading scale, a +/// monospace face for code, the three emphasis marks, and one left margin per +/// block-quote nesting depth. +class StyleRegistry final { +public: + StyleRegistry(); + + /// Throws if the index has no style. + [[nodiscard]] const TextStyle &text_style(std::uint32_t index) const; + [[nodiscard]] const ParagraphStyle & + paragraph_style(std::uint32_t index) const; + + /// @throws std::out_of_range unless @p level is 1 to 6. + [[nodiscard]] std::uint32_t heading_style(std::uint32_t level) const; + [[nodiscard]] std::uint32_t monospace_style() const; + [[nodiscard]] std::uint32_t emphasis_style() const; + [[nodiscard]] std::uint32_t strong_style() const; + [[nodiscard]] std::uint32_t strikethrough_style() const; + + /// The style of a paragraph nested in @p depth block quotes, interned on + /// first use; the default style for depth 0. + std::uint32_t quote_style(std::uint32_t depth); + +private: + std::vector m_text_styles; + std::vector m_paragraph_styles; +}; + +} // namespace odr::internal::markdown diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 68a4e2606..c38e65bb7 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +189,13 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoJsonFile(); } + if (as == FileType::markdown) { + ODR_VERBOSE(logger, "open as markdown"); + // Nothing rejects: md4c is total, so there is no detection to fail here. + auto text = std::make_shared(file); + return std::make_unique(std::move(text)); + } + if (as == FileType::xml) { ODR_VERBOSE(logger, "open as xml"); try { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b516952c..ffa7123f6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,8 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/markdown/markdown_file_test.cpp" + "src/internal/odf/odf_table_test.cpp" "src/internal/oldms/doc_test.cpp" diff --git a/test/src/internal/markdown/markdown_file_test.cpp b/test/src/internal/markdown/markdown_file_test.cpp new file mode 100644 index 000000000..739575739 --- /dev/null +++ b/test/src/internal/markdown/markdown_file_test.cpp @@ -0,0 +1,310 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include + +using namespace odr; +using namespace odr::internal; + +namespace { + +/// An `Element` holds a bare pointer into its document, so every test binds the +/// document to a local before walking it. +Document document(const std::string &markdown) { + const DecodedFile file(File::from_memory(markdown), FileType::markdown); + return file.as_document_file().document(); +} + +Element root(const Document &document) { return document.root_element(); } + +/// The direct children of an element, so a test can index into them. +std::vector children(const Element element) { + std::vector result; + for (const Element child : element.children()) { + result.push_back(child); + } + return result; +} + +/// Every text below @p element, concatenated — the reading text of a block, +/// whatever spans and links it is broken into. +std::string text_of(const Element element) { + if (element.type() == ElementType::text) { + return element.as_text().content(); + } + std::string result; + for (const Element child : element.children()) { + result += text_of(child); + } + return result; +} + +std::vector types_of(const std::vector &elements) { + std::vector result; + result.reserve(elements.size()); + for (const Element element : elements) { + result.push_back(element.type()); + } + return result; +} + +} // namespace + +TEST(MarkdownFile, a_markdown_file_is_a_text_document) { + const DecodedFile file(File::from_memory("# hello"), FileType::markdown); + const Document md = document("# hello"); + + EXPECT_EQ(file.file_type(), FileType::markdown); + EXPECT_EQ(file.file_category(), FileCategory::document); + EXPECT_TRUE(file.is_document_file()); + EXPECT_FALSE(file.is_text_file()); + EXPECT_EQ(file.file_meta().mimetype, "text/markdown"); + EXPECT_EQ(file.as_document_file().document_type(), DocumentType::text); + EXPECT_EQ(md.document_type(), DocumentType::text); + EXPECT_EQ(root(md).type(), ElementType::root); +} + +/// Markdown has no signature, and a content probe for it is a probe for prose, +/// so the caller routes on the file name and a `.md` still opens as text. +TEST(MarkdownFile, it_is_not_detected_by_content) { + const File file = File::from_memory("# hello\n\nsome *markdown*\n"); + + EXPECT_THAT(DecodedFile::list_file_types(file), + testing::Not(testing::Contains(FileType::markdown))); + EXPECT_EQ(DecodedFile(file).file_type(), FileType::text_file); +} + +/// Nothing rejects: any UTF-8 byte sequence is some markdown document. +TEST(MarkdownFile, anything_decodable_reads_as_markdown) { + EXPECT_NO_THROW(std::ignore = document("*")); + EXPECT_NO_THROW(std::ignore = document("]]] not markup [[[")); + + const Document empty = document("\n"); + EXPECT_FALSE(root(empty).first_child()); +} + +/// `Text::content()` is UTF-8 to every binding, so bytes we cannot decode have +/// no document at all — the text rendering path stays open to them. +TEST(MarkdownFile, an_undecodable_encoding_has_no_document) { + const File file = File::from_memory("# hello"); + const internal::markdown::MarkdownFile markdown_file( + std::make_shared(file.impl(), + TextEncoding::shift_jis)); + + EXPECT_FALSE(markdown_file.is_decodable()); + EXPECT_THROW(std::ignore = markdown_file.document(), UnsupportedTextEncoding); +} + +TEST(MarkdownDocument, a_paragraph_is_a_paragraph) { + const Document md = document("one\ntwo\n\nthree\n"); + const std::vector blocks = children(root(md)); + + ASSERT_EQ(blocks.size(), 2); + EXPECT_EQ(types_of(blocks), + (std::vector{ElementType::paragraph, ElementType::paragraph})); + // a soft break is a space, not a break + EXPECT_EQ(text_of(blocks[0]), "one two"); + EXPECT_EQ(text_of(blocks[1]), "three"); +} + +TEST(MarkdownDocument, a_hard_break_is_a_line_break) { + const Document md = document("one \ntwo\n"); + const std::vector parts = children(children(root(md))[0]); + + EXPECT_EQ(types_of(parts), + (std::vector{ElementType::text, ElementType::line_break, + ElementType::text})); +} + +/// The model has no heading, so a heading is a paragraph whose text style +/// carries the level — the same compromise `odf_parser` makes for `text:h`. +TEST(MarkdownDocument, a_heading_is_a_paragraph_with_a_heading_style) { + const Document md = document("# one\n\n### three\n"); + const std::vector blocks = children(root(md)); + + ASSERT_EQ(blocks.size(), 2); + EXPECT_EQ(types_of(blocks), + (std::vector{ElementType::paragraph, ElementType::paragraph})); + + const TextStyle h1 = blocks[0].as_paragraph().text_style(); + const TextStyle h3 = blocks[1].as_paragraph().text_style(); + + EXPECT_EQ(h1.font_weight, FontWeight::bold); + EXPECT_EQ(h3.font_weight, FontWeight::bold); + ASSERT_TRUE(h1.font_size.has_value()); + ASSERT_TRUE(h3.font_size.has_value()); + EXPECT_GT(h1.font_size->magnitude(), h3.font_size->magnitude()); + EXPECT_EQ(h1.font_size->unit().name(), "em"); +} + +TEST(MarkdownDocument, emphasis_becomes_a_span) { + const Document md = document("a *b* **c** ~~d~~\n"); + const std::vector parts = children(children(root(md))[0]); + + ASSERT_EQ(parts.size(), 6); + EXPECT_EQ(parts[1].type(), ElementType::span); + EXPECT_EQ(parts[1].as_span().style().font_style, FontStyle::italic); + EXPECT_EQ(parts[3].as_span().style().font_weight, FontWeight::bold); + EXPECT_EQ(parts[5].as_span().style().font_line_through, true); +} + +/// Nested emphasis nests spans, each carrying only its own mark. +TEST(MarkdownDocument, nested_emphasis_nests_spans) { + const Document md = document("***bold italic***\n"); + const Element outer = children(children(root(md))[0])[0]; + const Element inner = children(outer)[0]; + + EXPECT_EQ(outer.as_span().style().font_style, FontStyle::italic); + EXPECT_FALSE(outer.as_span().style().font_weight.has_value()); + EXPECT_EQ(inner.as_span().style().font_weight, FontWeight::bold); + EXPECT_EQ(text_of(outer), "bold italic"); +} + +TEST(MarkdownDocument, inline_code_is_a_monospace_span) { + const Document md = document("a `b` c\n"); + const Element span = children(children(root(md))[0])[1]; + + EXPECT_EQ(span.type(), ElementType::span); + EXPECT_EQ(span.as_span().style().font_name, "monospace"); + EXPECT_EQ(text_of(span), "b"); +} + +/// A code block is one monospace paragraph per line: the model has no +/// pre-formatted block, and one paragraph would collapse the newlines. +TEST(MarkdownDocument, a_code_block_is_one_paragraph_per_line) { + const Document md = document("```\none\n\n three\n```\n"); + const Element group = children(root(md))[0]; + const std::vector lines = children(group); + + EXPECT_EQ(group.type(), ElementType::group); + ASSERT_EQ(lines.size(), 3); + EXPECT_EQ(text_of(lines[0]), "one"); + EXPECT_EQ(text_of(lines[1]), ""); + EXPECT_EQ(text_of(lines[2]), " three"); + EXPECT_EQ(lines[0].as_paragraph().text_style().font_name, "monospace"); +} + +TEST(MarkdownDocument, a_link_carries_its_href) { + const Document md = document("see [here](https://a.example/b).\n"); + const Element link = children(children(root(md))[0])[1]; + + EXPECT_EQ(link.type(), ElementType::link); + EXPECT_EQ(link.as_link().href(), "https://a.example/b"); + EXPECT_EQ(text_of(link), "here"); +} + +TEST(MarkdownDocument, an_unordered_list) { + const Document md = document("- a\n- b\n"); + const Element list = children(root(md))[0]; + const std::vector items = children(list); + + EXPECT_EQ(list.as_list().type(), ListType::unordered); + ASSERT_EQ(items.size(), 2); + EXPECT_EQ(items[0].as_list_item().marker(), "•"); + EXPECT_FALSE(items[0].as_list_item().number().has_value()); + EXPECT_EQ(text_of(items[1]), "b"); +} + +TEST(MarkdownDocument, an_ordered_list_counts_from_its_start) { + const Document md = document("3. a\n4. b\n"); + const Element list = children(root(md))[0]; + const std::vector items = children(list); + + EXPECT_EQ(list.as_list().type(), ListType::ordered); + ASSERT_EQ(items.size(), 2); + EXPECT_EQ(items[0].as_list_item().marker(), "3."); + EXPECT_EQ(items[0].as_list_item().number(), 3); + EXPECT_EQ(items[1].as_list_item().marker(), "4."); + EXPECT_EQ(items[1].as_list_item().number(), 4); +} + +/// The model has no checkbox, so the box replaces the item's marker. +TEST(MarkdownDocument, a_task_list_item_is_marked_with_its_box) { + const Document md = document("- [ ] a\n- [x] b\n"); + const std::vector items = children(children(root(md))[0]); + + ASSERT_EQ(items.size(), 2); + EXPECT_EQ(items[0].as_list_item().marker(), "☐"); + EXPECT_EQ(items[1].as_list_item().marker(), "☑"); +} + +/// A quote is a group — the renderer writes no wrapper for one — so what +/// survives is the left margin its paragraphs carry, one step per level. +TEST(MarkdownDocument, a_block_quote_indents_its_paragraphs) { + const Document md = document("> one\n>\n> > two\n"); + const Element quote = children(root(md))[0]; + const std::vector inner = children(quote); + + EXPECT_EQ(quote.type(), ElementType::group); + ASSERT_EQ(inner.size(), 2); + + const ParagraphStyle one = inner[0].as_paragraph().style(); + const ParagraphStyle two = children(inner[1])[0].as_paragraph().style(); + + ASSERT_TRUE(one.margin.left.has_value()); + ASSERT_TRUE(two.margin.left.has_value()); + EXPECT_LT(one.margin.left->magnitude(), two.margin.left->magnitude()); +} + +TEST(MarkdownDocument, a_gfm_table) { + const Document md = document("| a | b |\n" + "|---|--:|\n" + "| 1 | 2 |\n"); + const Element table = children(root(md))[0]; + + ASSERT_EQ(table.type(), ElementType::table); + EXPECT_EQ(table.as_table().dimensions().rows, 2); + EXPECT_EQ(table.as_table().dimensions().columns, 2); + + std::vector rows; + for (const Element row : table.as_table().rows()) { + rows.push_back(row); + } + ASSERT_EQ(rows.size(), 2); + + const std::vector header = children(rows[0]); + ASSERT_EQ(header.size(), 2); + EXPECT_EQ(text_of(header[0]), "a"); + // a header cell is bold, on the span the renderer takes the weight from + const Element header_span = children(children(header[0])[0])[0]; + EXPECT_EQ(header_span.as_span().style().font_weight, FontWeight::bold); + EXPECT_EQ(header[1].as_table_cell().style().horizontal_align, + HorizontalAlign::right); + EXPECT_EQ(text_of(children(rows[1])[1]), "2"); +} + +TEST(MarkdownDocument, entities_are_resolved) { + const Document md = document("& A B ¬anentity;\n"); + + EXPECT_EQ(text_of(root(md)), "& A B ¬anentity;"); +} + +/// There is no passthrough element in the model, so raw html goes nowhere. +TEST(MarkdownDocument, raw_html_is_dropped) { + const Document inline_html = document("a bold c\n"); + const Document block_html = document("
\nblock\n
\n"); + + EXPECT_EQ(text_of(root(inline_html)), "a bold c"); + EXPECT_EQ(text_of(root(block_html)), ""); +} + +/// Stage 4: an image is not a frame yet, and its alt text is what is left. +TEST(MarkdownDocument, an_image_leaves_its_alt_text_behind) { + const Document md = document("![a diagram](d.svg)\n"); + + EXPECT_EQ(text_of(root(md)), "a diagram"); +}