diff --git a/AGENTS.md b/AGENTS.md index d7b27100..0eafe31f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/odf/` | OpenDocument (odt/ods/odp/odg); see [`odf/AGENTS.md`](src/odr/internal/odf/AGENTS.md). | | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | +| `src/odr/internal/iwork/` | Apple iWork (`.pages` today); see [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) + [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md). | | `src/odr/internal/pdf/` | PDF (own parser). | | `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | @@ -218,6 +219,15 @@ Dispatch `release.yml` against main, publish the draft that appears — 4. Register the factory (e.g. `oldms_file.cpp::document()` switches on `file_type()`), add sources to `CMakeLists.txt`, add a GoogleTest. +## Apple iWork (`iwork`) + +`.pages` opens as a text document and renders its body text; `.numbers` and +`.key` are named but not decoded. There is no spec — the module cites fixtures +instead, keeps its own Snappy and protobuf readers, and fails soft on archive +types it has not mapped. Read [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) +before touching it, and [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md) for +what comes next. + ## Legacy Microsoft binary formats (`oldms`) CFB container handling exists; each format is a small module under `oldms/` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc85b98..678452ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- Apple iWork: a `.pages` file opens as a text document and renders its body + text, instead of coming back as the zip it is made of. Styles, page geometry, + images and tables are not read yet. `.numbers` and `.key` are named — + `FileType::iwork_numbers`, `FileType::iwork_keynote`, their extensions and + MIME types — but there is no decoder behind either. + ## v6.10.1 - 2026-08-21 - A linked image in a docx or xlsx (`embed_images = false`) is named relative diff --git a/CMakeLists.txt b/CMakeLists.txt index 04feb219..ab090b17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,14 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_archive.cpp" + "src/odr/internal/iwork/iwork_document.cpp" + "src/odr/internal/iwork/iwork_element_registry.cpp" + "src/odr/internal/iwork/iwork_file.cpp" + "src/odr/internal/iwork/iwork_parser.cpp" + "src/odr/internal/iwork/iwork_protobuf.cpp" + "src/odr/internal/iwork/iwork_snappy.cpp" + "src/odr/internal/json/json_file.cpp" "src/odr/internal/json/json_util.cpp" diff --git a/apple/include/OdrCoreObjC/ODRFile.h b/apple/include/OdrCoreObjC/ODRFile.h index d1b07078..ac19d087 100644 --- a/apple/include/OdrCoreObjC/ODRFile.h +++ b/apple/include/OdrCoreObjC/ODRFile.h @@ -83,6 +83,10 @@ typedef NS_ENUM(NSInteger, ODRFileType) { ODRFileTypeEnhancedMetafile, ODRFileTypeXml, + + ODRFileTypeIworkPages, + ODRFileTypeIworkNumbers, + ODRFileTypeIworkKeynote, } NS_SWIFT_NAME(FileType); typedef NS_ENUM(NSInteger, ODRFileCategory) { diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index 472f05f8..d616b1ab 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -92,6 +92,10 @@ ODR_SAME_ENUM(ODRFileTypeEnhancedMetafile, odr::FileType::enhanced_metafile); ODR_SAME_ENUM(ODRFileTypeXml, odr::FileType::xml); +ODR_SAME_ENUM(ODRFileTypeIworkPages, odr::FileType::iwork_pages); +ODR_SAME_ENUM(ODRFileTypeIworkNumbers, odr::FileType::iwork_numbers); +ODR_SAME_ENUM(ODRFileTypeIworkKeynote, odr::FileType::iwork_keynote); + ODR_SAME_ENUM(ODRFileCategoryUnknown, odr::FileCategory::unknown); ODR_SAME_ENUM(ODRFileCategoryText, odr::FileCategory::text); ODR_SAME_ENUM(ODRFileCategoryImage, odr::FileCategory::image); diff --git a/jni/java/app/opendocument/core/FileType.java b/jni/java/app/opendocument/core/FileType.java index 80d13963..74b1f5e1 100644 --- a/jni/java/app/opendocument/core/FileType.java +++ b/jni/java/app/opendocument/core/FileType.java @@ -16,7 +16,8 @@ public enum FileType { OGG_AUDIO, WAVEFORM_AUDIO, FREE_LOSSLESS_AUDIO_CODEC, MPEG4_VIDEO, QUICKTIME_VIDEO, THIRD_GENERATION_PARTNERSHIP_VIDEO, MATROSKA_VIDEO, AUDIO_VIDEO_INTERLEAVE, SCALABLE_VECTOR_GRAPHICS, WINDOWS_ICON, JPEG_XL, - JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML; + JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML, + IWORK_PAGES, IWORK_NUMBERS, IWORK_KEYNOTE; static FileType fromNative(int code) { return code < 0 ? null : values()[code]; diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index ac48ef54..0d1163d8 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -83,7 +83,10 @@ void odr_python::bind_file(py::module_ &m) { .value("photoshop_document", odr::FileType::photoshop_document) .value("windows_metafile", odr::FileType::windows_metafile) .value("enhanced_metafile", odr::FileType::enhanced_metafile) - .value("xml", odr::FileType::xml); + .value("xml", odr::FileType::xml) + .value("iwork_pages", odr::FileType::iwork_pages) + .value("iwork_numbers", odr::FileType::iwork_numbers) + .value("iwork_keynote", odr::FileType::iwork_keynote); py::enum_(m, "FileCategory") .value("unknown", odr::FileCategory::unknown) diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index aaa8d361..c94619a8 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -68,6 +68,8 @@ NoFontFile::NoFontFile() : Exception("not a font file") {} NoLegacyMicrosoftFile::NoLegacyMicrosoftFile() : Exception("not a legacy microsoft office file") {} +NoIworkFile::NoIworkFile() : Exception("not an iwork file") {} + NoXmlFile::NoXmlFile() : Exception("not an xml file") {} NoSvgFile::NoSvgFile() : Exception("not an svg file") {} diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index c9d55fc1..597bc863 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -142,6 +142,11 @@ struct NoLegacyMicrosoftFile final : Exception { NoLegacyMicrosoftFile(); }; +/// @brief No iWork file exception +struct NoIworkFile final : Exception { + NoIworkFile(); +}; + /// @brief No XML file exception struct NoXmlFile final : Exception { NoXmlFile(); diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 45aea937..930be535 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -149,6 +149,14 @@ enum class FileType { // `[text_file, xml, scalable_vector_graphics]`. // https://en.wikipedia.org/wiki/XML xml, + + // https://en.wikipedia.org/wiki/IWork + iwork_pages, + // Classification only - `.numbers` and `.key` sit in the same package the + // pages engine reads, but which app wrote one is read off its root archive + // and no fixture pins those two, so nothing detects or decodes them yet. + iwork_numbers, + iwork_keynote, }; /// @brief Collection of file categories. diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 61c6ecdd..5333423e 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -250,6 +250,24 @@ constexpr std::array avi_mimetypes{"video/x-msvideo"sv, "video/avi"sv, // `decrypt` on an OOXML document type means a password-protected package, // detected as `office_open_xml_encrypted` and decrypting into the type named // here. ODF files decrypt in place and keep their type. +constexpr std::array pages_extensions{"pages"sv}; +constexpr std::array pages_mimetypes{ + "application/vnd.apple.pages"sv, + "application/x-iwork-pages-sffpages"sv, +}; + +constexpr std::array numbers_extensions{"numbers"sv}; +constexpr std::array numbers_mimetypes{ + "application/vnd.apple.numbers"sv, + "application/x-iwork-numbers-sffnumbers"sv, +}; + +constexpr std::array keynote_extensions{"key"sv}; +constexpr std::array keynote_mimetypes{ + "application/vnd.apple.keynote"sv, + "application/x-iwork-keynote-sffkey"sv, +}; + constexpr std::array table{ Row{FileType::unknown, "unknown"sv, @@ -755,6 +773,33 @@ constexpr std::array table{ .open = true, .translate_html = true, .color_scheme = true}}, + + Row{FileType::iwork_pages, + "pages"sv, + pages_extensions, + pages_mimetypes, + FileCategory::document, + DocumentType::text, + {.detect_by_content = true, + .open = true, + .translate_html = true, + .color_scheme = true}}, + // Classified so a caller can name these two and hand their MIME types to a + // file picker; no engine reads either yet. + Row{FileType::iwork_numbers, + "numbers"sv, + numbers_extensions, + numbers_mimetypes, + FileCategory::document, + DocumentType::spreadsheet, + {}}, + Row{FileType::iwork_keynote, + "key"sv, + keynote_extensions, + keynote_mimetypes, + FileCategory::document, + DocumentType::presentation, + {}}, }; /// Finds the row whose list, selected by @p list, contains @p needle. diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md new file mode 100644 index 00000000..4223d55f --- /dev/null +++ b/src/odr/internal/iwork/AGENTS.md @@ -0,0 +1,109 @@ +# AGENTS.md — `internal/iwork` + +Read the root [`AGENTS.md`](../../../../AGENTS.md) first, then +[`PLAN.md`](PLAN.md), which is where this module is going and in what order. +This file is what the landed stages decided, and why. + +Landed: **stage 1** (detection and the container) and **stage 2** (Pages body +text). A `.pages` opens as a text document and renders its paragraphs. +Everything else in `PLAN.md` is still ahead. + +## There is no spec, so a fixture is the citation + +Apple has never published the `.proto` schemas and nothing is vendored under +`offline/documentation/`. Where `oldms/` writes `[MS-XLS] §2.4.1`, this module +writes `empty.pages Index/Document.iwa +0` — the byte layout verified against a +file in the repo is the only claim treated as fact. + +Everything the engine reads by number lives in `iwork_types.hpp`, each constant +cited to the fixture it was read off. Read `numbers-parser`, `keynote-parser`, +`obriensp/iWorkFileFormat` and `libetonyek` for facts; **copy code from none of +them**. + +**Fail soft on a type id we have not mapped, fail fast on broken framing.** The +root `AGENTS.md` says to throw where the spec dictates what to expect. Here +there is no spec, and an unknown type id means Apple shipped a version we have +not seen — a reader that throws on one cannot open next year's files. What does +throw: framing that overruns the file, a Snappy block that does not fill its +declared length, a varint that does not terminate, an identifier the package +does not hold, and text that is not UTF-8. + +## No new dependencies + +Two pieces would normally be a conan line each, and both would be wrong. + +- **Snappy** — the `.iwa` framing is Apple's own (`0x00`, a little-endian + 24-bit compressed length, repeated to EOF), not Snappy's stream framing, so + only the *block* decoder applies. `iwork_snappy.cpp` is that, in about a + hundred lines. +- **Protobuf** — only the wire format is needed, and with no schemas a code + generator has nothing to generate. Linking conan `protobuf` would drag it + into the wasm, android and apple builds to replace `iwork_protobuf.cpp`. + +Both stay inside `iwork/` until something else wants them; a wire reader with +one user has not earned a package. + +## `Message` views the buffer it was read from + +`iwork::Message` parses one level eagerly and leaves nested messages, strings +and packed fields as `std::string_view`s into the bytes it was handed. So the +buffer has to outlive it — `Component` owns its decompressed data behind a +`unique_ptr` for exactly that reason, and a `Message(some_temporary())` is a +dangling read rather than a compile error. + +## An `.iwa` is an object graph, not a tree + +A component file is a flat sequence of `(varint length, TSP.ArchiveInfo, +payload)`, and objects reference each other by identifier — across components. +So `Package` reads the component list from `Index/Metadata.iwa` first and +decompresses a component when something in it is asked for; `object(id)` loads +further components until the identifier turns up. Walking files in directory +order and hoping a tree falls out is the mistake to avoid. + +**Component names are not file names.** `Index/Metadata.iwa` maps a component's +name to its locator, and the locator carries an identifier suffix often enough +that globbing for `CalculationEngine.iwa` finds it in one fixture and not in +the other. + +The one place that skips the component list is detection: `IworkFile` reads +`/Index/Document.iwa` directly, because it runs on every zip a caller opens and +`Document` is the component whose file name never carries a suffix. + +## Which app wrote the package comes off the root archive + +`TP.DocumentArchive` is type 10000, verified on both `.pages` fixtures. The +extension is not consulted — a caller may have lost it — and neither is +`Metadata/Properties.plist`, which names an app version but not the app. + +That is also why only `.pages` is detected. `iwork_numbers` and `iwork_keynote` +have `file_type_table.cpp` rows so a caller can name them and hand a file +picker their MIME types, but no capabilities: reading their root archive types +off a guess is exactly what this module does not do, and neither has a fixture +in the test data yet. + +## Paragraphs come from the run table + +A `TSWP.StorageArchive` holds its text as a few large strings plus run tables +parallel to it — index/value pairs for paragraph styles, character styles and +attachments. Paragraph boundaries are the **paragraph style table's**, not +every `\n` in the text. The two agree on both fixtures, but the table is what +says so, and `U+2028` is a line break *inside* a paragraph rather than a +paragraph boundary. + +Run tables count in **UTF-16 code units** while the text is UTF-8, so +`iwork_parser.cpp` translates the indices in one pass over the text. An index +that lands mid-character is an error, not a rounding. + +`U+FFFC` is where a drawable is anchored. Nothing reads drawables yet, so the +anchor is dropped rather than rendered as a glyph — see stage 4. + +`empty.pages` is the regression that matters at this level: a body storage that +carries no text at all must produce an empty body, not an exception. + +## Not read yet + +`Index/DocumentStylesheet.iwa` (so `text_root_page_layout` is empty and every +style is the default), drawables and images, `Index/Tables/`, and everything +`PLAN.md` lists as deferred. `password_encrypted()` is not answered either: an +encrypted package is one whose `Index/Document.iwa` does not decompress, which +falls back to reporting the file as a zip. diff --git a/src/odr/internal/iwork/PLAN.md b/src/odr/internal/iwork/PLAN.md index f57831fc..a8bbf830 100644 --- a/src/odr/internal/iwork/PLAN.md +++ b/src/odr/internal/iwork/PLAN.md @@ -1,28 +1,24 @@ # iWork plan -Where an iwork module would go, and in what order. Written before stage 1; keep -it honest as stages land. +Where an iwork module goes, and in what order. Written before stage 1; kept +honest as stages land. **Stages 1 and 2 have landed** — see +[`AGENTS.md`](AGENTS.md) for what they decided. Stage 3 is next. ## Today -Nothing decodes, and unlike rtf there is not even a `FileType` yet. A `.pages` -is a zip, so `magic.cpp:95` reports `FileType::zip`, `list_file_types` probes -odf then ooxml (`open_strategy.cpp:213-238`), both fail, and the caller gets -`[zip]`. `odr::open` hands back a `zip::ZipFile` — an archive, not a document. +A `.pages` opens as a text document and renders its body text. `.numbers` and +`.key` have `FileType` entries and `file_type_table.cpp` rows so a caller can +name them, but no capabilities and no engine behind them: which app wrote a +package is read off its root archive type, and neither has a fixture to pin +that against. -Two fixtures are already committed: +Two fixtures are committed: `test/data/input/odr-public/pages/{empty.pages,style-various-1.pages}`, both written by iWork 13.2 (`Metadata/BuildVersionHistory.plist`). Neither is listed -in `index.csv` and neither has reference output, so nothing exercises them. -`style-various-1.pages` carries `Index/Tables/` and nine files under `Data/`, -which is most of the surface below. - -New `FileType` entries append at the end of the enum — `file.hpp:98` says so — -and **the bindings do need updating** here, unlike rtf: `python/src/bind_file.cpp`, -`jni/java/app/opendocument/core/FileType.java`, -`apple/include/OdrCoreObjC/ODRFile.h` + `apple/src/ODRFile.mm`. Wasm does not: -it derives its enums from `odr::all_file_types()` at runtime -(`wasm/src/wasm_core.cpp:35`, `:69`). +in `index.csv` — they do not need to be, `TestData` picks up anything the file +type table knows an extension for — and they gained reference output when stage +2 turned `translate_html` on. `style-various-1.pages` carries `Index/Tables/` +and nine files under `Data/`, which is most of the surface below. ## Spec @@ -145,11 +141,26 @@ silently. --- -## Stage 1 — detection and the container +## Stage 1 — detection and the container *(landed)* Nothing renders yet. The point is that the bytes come apart correctly and the type is reported, which is also the whole of what a file picker needs. +Landed as planned, with three deviations: + +- **Only `iwork_pages` detects and opens.** `iwork_numbers` and `iwork_keynote` + are classification-only rows, because the root archive type of a `.numbers` + or a `.key` cannot be read off a fixture that does not exist, and this module + does not guess. +- **`password_encrypted()` is not answered.** `Index/Metadata.iwph` was going + to report it, but nothing here has ever seen an encrypted package; an + encrypted one is one whose `Index/Document.iwa` does not decompress, and it + falls back to being reported as a zip. +- **Detection does not read `Index/Metadata.iwa`.** It reads + `/Index/Document.iwa` straight, since it runs on every zip a caller opens and + `Document` is the one component whose file name never carries a suffix. The + component list is read when the document is. + - `iwork_snappy.{hpp,cpp}` — Apple framing plus block decompression, over the `std::istream *` / `std::streambuf *` shape `pdf::ObjectParser` uses (`pdf_object_parser.hpp`). @@ -183,7 +194,7 @@ type ID that must be skipped rather than thrown on, and framing truncated mid-block. Only the type-reporting test needs the fixtures — the data repos are fetched and optional, so everything that can be inline is. -## Stage 2 — Pages text +## Stage 2 — Pages text *(landed)* - walk from the document archive to the body's text storage (`TSWP.StorageArchive` in the reverse-engineering literature; confirm the type @@ -199,6 +210,13 @@ fetched and optional, so everything that can be inline is. produce an empty body and not an exception. - table row: `iwork_pages` gains `.translate_html = true`. +Landed as planned. What the fixtures settled: the body storage is field 4 of +`TP.DocumentArchive` (type 10000) and is a `TSWP.StorageArchive` (type 2001); +its paragraph style table is field 5, a `TSWP.ObjectAttributeTable` whose +field 1 repeats the entries — so the run tables are one level deeper than +"repeated entries on the storage". Run-table indices count UTF-16 code units +against UTF-8 text, which the parser translates in one pass. + ## Stage 3 — Pages styles - `Index/DocumentStylesheet.iwa`. Style archives are sparse property sets with a @@ -286,10 +304,11 @@ without a Numbers fixture existing. ## Test data -`empty.pages` and `style-various-1.pages` are already in -`test/data/input/odr-public/pages/` but absent from `index.csv` and from -reference output. Add them to the index in stage 1, and regenerate reference -output when stage 2 flips `translate_html` on. +`empty.pages` and `style-various-1.pages` are in +`test/data/input/odr-public/pages/`. They need no `index.csv` row — +`TestData::test_files` picks up any file whose extension the file type table +knows — and reference output was regenerated when stage 2 flipped +`translate_html` on. Stages 5 and 7 each need a fixture that does not exist yet — one `.key` and one `.numbers` in the public repo. Everything at container level stays inline, per diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp new file mode 100644 index 00000000..487ddcd1 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -0,0 +1,180 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// Field numbers of `TSP.PackageMetadata` and the `ComponentInfo` it repeats, +/// read off `empty.pages Index/Metadata.iwa` (object 2, type 11006). +constexpr std::uint32_t package_metadata_components = 3; +constexpr std::uint32_t component_info_identifier = 1; +constexpr std::uint32_t component_info_preferred_locator = 2; +constexpr std::uint32_t component_info_locator = 3; + +/// Field numbers of `TSP.ArchiveInfo` and the `MessageInfo` it repeats. +constexpr std::uint32_t archive_info_identifier = 1; +constexpr std::uint32_t archive_info_messages = 2; +constexpr std::uint32_t message_info_type = 1; +constexpr std::uint32_t message_info_length = 3; + +AbsPath component_path(const std::string &locator) { + return AbsPath("/Index").join(RelPath(locator + ".iwa")); +} + +} // namespace + +std::string iwork::read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path) { + const std::shared_ptr file = filesystem.open(path); + if (!file) { + throw std::runtime_error("iwork: missing " + path.string()); + } + const std::unique_ptr stream = file->stream(); + return iwa_decompress(util::stream::read(*stream)); +} + +std::vector iwork::read_objects(const std::string_view data) { + std::vector result; + + std::size_t position = 0; + while (position < data.size()) { + const std::uint64_t info_length = read_varint(data, position); + if (info_length > data.size() - position) { + throw std::runtime_error("iwork: archive info runs past the component"); + } + const Message info(data.substr(position, info_length)); + position += info_length; + + Object object; + object.identifier = info.number_field(archive_info_identifier).value_or(0); + + // the payload holds every message the info names, back to back; only the + // first is modelled, the length of the rest is what skips them + std::size_t payload_length = 0; + std::size_t first_length = 0; + bool first = true; + for (const Field &message : info.repeated_field(archive_info_messages)) { + if (message.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed message info"); + } + const Message message_info(message.bytes); + const std::uint64_t length = + message_info.number_field(message_info_length).value_or(0); + if (first) { + object.type = static_cast( + message_info.number_field(message_info_type).value_or(0)); + first_length = length; + first = false; + } + payload_length += length; + } + + if (payload_length > data.size() - position) { + throw std::runtime_error("iwork: object payload runs past the component"); + } + object.payload = data.substr(position, first_length); + position += payload_length; + + result.push_back(object); + } + + return result; +} + +iwork::Component::Component(std::string locator, std::string data) + : m_locator{std::move(locator)}, + m_data{std::make_unique(std::move(data))}, + m_objects{read_objects(*m_data)} {} + +const std::string &iwork::Component::locator() const noexcept { + return m_locator; +} + +const std::vector &iwork::Component::objects() const noexcept { + return m_objects; +} + +iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) + : m_filesystem{&filesystem} { + const std::string data = read_iwa(filesystem, AbsPath("/Index/Metadata.iwa")); + const std::vector objects = read_objects(data); + if (objects.empty()) { + throw std::runtime_error("iwork: empty package metadata"); + } + + const Message metadata(objects.front().payload); + for (const Field &component : + metadata.repeated_field(package_metadata_components)) { + if (component.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed component info"); + } + const Message info(component.bytes); + + ComponentInfo result; + result.identifier = + info.number_field(component_info_identifier).value_or(0); + result.name = std::string(info.bytes_field(component_info_preferred_locator) + .value_or(std::string_view())); + result.locator = std::string( + info.bytes_field(component_info_locator).value_or(result.name)); + if (result.name.empty()) { + throw std::runtime_error("iwork: component without a name"); + } + m_component_infos.push_back(std::move(result)); + } +} + +const iwork::Component &iwork::Package::component(const std::string &name) { + const auto it = + std::ranges::find(m_component_infos, name, &ComponentInfo::name); + if (it == std::ranges::end(m_component_infos)) { + throw std::runtime_error("iwork: no component named " + name); + } + return load_(*it); +} + +const iwork::Object &iwork::Package::object(const std::uint64_t identifier) { + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + + for (const ComponentInfo &info : m_component_infos) { + load_(info); + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + } + + throw std::runtime_error("iwork: no object " + std::to_string(identifier)); +} + +const iwork::Component &iwork::Package::load_(const ComponentInfo &info) { + // by locator, not by name: a name is shared across components, so keying on + // it would hand back the wrong file and leave the other never loaded + if (const auto it = + std::ranges::find(m_components, info.locator, &Component::locator); + it != std::ranges::end(m_components)) { + return *it; + } + + const Component &component = m_components.emplace_back( + info.locator, read_iwa(*m_filesystem, component_path(info.locator))); + for (const Object &object : component.objects()) { + m_objects.emplace(object.identifier, &object); + } + return component; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp new file mode 100644 index 00000000..7a286e01 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal { +class AbsPath; +} // namespace odr::internal + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// One archived object. `TSP.ArchiveInfo` names its identifier (field 1) and, +/// per payload message, a `MessageInfo` (field 2) carrying the message type +/// and its length. An object usually holds one message; where it holds more, +/// only the first is modelled. +/// +/// Verified on `empty.pages Index/Document.iwa +0`: `08 01` (identifier 1), +/// `12 52` (an 82-byte `MessageInfo`), `08 90 4e` (type 10000), `18 e0 0c` +/// (payload length 1632). +struct Object final { + std::uint64_t identifier{}; + std::uint32_t type{}; + std::string_view payload; +}; + +/// The objects of one `.iwa`, over the bytes it decompressed to. Object +/// payloads are views into those bytes. +class Component final { +public: + Component(std::string locator, std::string data); + + /// The file the component was loaded from, without `/Index/` and `.iwa`. + /// Unlike its name, this is unique across the package. + [[nodiscard]] const std::string &locator() const noexcept; + [[nodiscard]] const std::vector &objects() const noexcept; + +private: + std::string m_locator; + std::unique_ptr m_data; + std::vector m_objects; +}; + +/// An iWork package: the component list from `Index/Metadata.iwa`, and the +/// components loaded from it so far. +/// +/// Objects reference each other by identifier across components, so the list +/// is read first and a component is decompressed when something in it is +/// asked for. +class Package final { +public: + explicit Package(const abstract::ReadableFilesystem &filesystem); + + /// The first component named @p name in the package's component list — a + /// name is not unique, `Tables/DataList` names dozens. Throws when the + /// package holds none. + const Component &component(const std::string &name); + + /// The object @p identifier names, loading components until it is found. + /// Throws when no component holds it. + const Object &object(std::uint64_t identifier); + +private: + /// One entry of `TSP.PackageMetadata`'s component list: the identifier of + /// the component's root object, the name it is known by, and the file it + /// lives in — which carries an identifier suffix often enough that the file + /// name is not a way to find it. + struct ComponentInfo final { + std::uint64_t identifier{}; + std::string name; + std::string locator; + }; + + const abstract::ReadableFilesystem *m_filesystem{nullptr}; + std::vector m_component_infos; + std::deque m_components; + std::unordered_map m_objects; + + const Component &load_(const ComponentInfo &info); +}; + +/// Reads @p path off @p filesystem and undoes its `.iwa` framing. +std::string read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path); + +/// Splits a decompressed `.iwa` into its objects, over @p data. +std::vector read_objects(std::string_view data); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_document.cpp b/src/odr/internal/iwork/iwork_document.cpp new file mode 100644 index 00000000..5d6848f7 --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.cpp @@ -0,0 +1,185 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace odr::internal::iwork { + +namespace { +std::unique_ptr +create_element_adapter(ElementRegistry ®istry); +} + +Document::Document(std::shared_ptr files) + : internal::Document(FileType::iwork_pages, DocumentType::text, + std::move(files)) { + m_root_element = parse_pages_tree(m_element_registry, *m_files); + + m_element_adapter = create_element_adapter(m_element_registry); +} + +const ElementRegistry &Document::element_registry() const { + return m_element_registry; +} + +bool Document::is_editable() const noexcept { return false; } + +bool Document::is_savable(const bool encrypted) const noexcept { + (void)encrypted; + return false; +} + +void Document::save(const Path &path) const { + (void)path; + throw UnsupportedOperation(); +} + +void Document::save(const Path &path, const char *password) const { + (void)path; + (void)password; + throw UnsupportedOperation(); +} + +namespace { + +class ElementAdapter final : public abstract::ElementAdapter, + public abstract::TextRootAdapter, + public abstract::LineBreakAdapter, + public abstract::ParagraphAdapter, + public abstract::TextAdapter { +public: + explicit ElementAdapter(ElementRegistry ®istry) : m_registry(®istry) {} + + [[nodiscard]] ElementType + element_type(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).type; + } + + [[nodiscard]] ElementIdentifier + element_parent(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).parent_id; + } + [[nodiscard]] ElementIdentifier + element_first_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).first_child_id; + } + [[nodiscard]] ElementIdentifier + element_last_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).last_child_id; + } + [[nodiscard]] ElementIdentifier + element_previous_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).previous_sibling_id; + } + [[nodiscard]] ElementIdentifier + element_next_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).next_sibling_id; + } + + [[nodiscard]] bool + element_is_unique(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_self_locatable(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_editable(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] DocumentPath + element_document_path(const ElementIdentifier element_id) const override { + return util::document::extract_path(*this, element_id, null_element_id); + } + [[nodiscard]] ElementIdentifier + element_navigate_path(const ElementIdentifier element_id, + const DocumentPath &path) const override { + return util::document::navigate_path(*this, element_id, path); + } + + [[nodiscard]] const TextRootAdapter * + text_root_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::root ? this : nullptr; + } + [[nodiscard]] const LineBreakAdapter * + line_break_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::line_break ? this : nullptr; + } + [[nodiscard]] const ParagraphAdapter * + paragraph_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::paragraph ? this : nullptr; + } + [[nodiscard]] const TextAdapter * + text_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::text ? this : nullptr; + } + + // The page geometry sits in the document archive and the styles in + // `Index/DocumentStylesheet.iwa`; neither is read yet. + [[nodiscard]] PageLayout + text_root_page_layout(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] ElementIdentifier text_root_first_master_page( + const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TextStyle + line_break_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] ParagraphStyle + paragraph_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] TextStyle + paragraph_text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] std::string + text_content(const ElementIdentifier element_id) const override { + return m_registry->text_element_at(element_id).text; + } + void text_set_content(const ElementIdentifier element_id, + const std::string &text) const override { + (void)element_id; + (void)text; + throw UnsupportedOperation(); + } + [[nodiscard]] TextStyle + text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + +private: + ElementRegistry *m_registry{nullptr}; +}; + +std::unique_ptr +create_element_adapter(ElementRegistry ®istry) { + return std::make_unique(registry); +} + +} // namespace + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_document.hpp b/src/odr/internal/iwork/iwork_document.hpp new file mode 100644 index 00000000..d73c8b0d --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include + +namespace odr::internal::iwork { + +/// A `.pages` package, read as a text document. +class Document final : public internal::Document { +public: + explicit Document(std::shared_ptr files); + + [[nodiscard]] const ElementRegistry &element_registry() const; + + [[nodiscard]] bool is_editable() const noexcept override; + [[nodiscard]] bool is_savable(bool encrypted) const noexcept override; + + void save(const Path &path) const override; + void save(const Path &path, const char *password) const override; + +private: + ElementRegistry m_element_registry; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.cpp b/src/odr/internal/iwork/iwork_element_registry.cpp new file mode 100644 index 00000000..2ce6952f --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.cpp @@ -0,0 +1,96 @@ +#include + +#include + +namespace odr::internal::iwork { + +void ElementRegistry::clear() noexcept { + m_elements.clear(); + m_texts.clear(); +} + +[[nodiscard]] std::size_t ElementRegistry::size() const noexcept { + return m_elements.size(); +} + +std::tuple +ElementRegistry::create_element(const ElementType type) { + Element &element = m_elements.emplace_back(); + ElementIdentifier element_id = m_elements.size(); + element.type = type; + return {element_id, element}; +} + +std::tuple +ElementRegistry::create_text_element() { + const auto &[element_id, element] = create_element(ElementType::text); + auto [it, success] = m_texts.emplace(element_id, Text{}); + return {element_id, element, it->second}; +} + +ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) { + check_element_id(id); + return m_elements.at(id - 1); +} + +ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) { + check_text_id(id); + return m_texts.at(id); +} + +const ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) const { + check_element_id(id); + return m_elements.at(id - 1); +} + +const ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) const { + check_text_id(id); + return m_texts.at(id); +} + +void ElementRegistry::append_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id) { + check_element_id(parent_id); + check_element_id(child_id); + if (element_at(child_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_child: child already has a parent"); + } + + const ElementIdentifier previous_sibling_id = + element_at(parent_id).last_child_id; + + element_at(child_id).parent_id = parent_id; + element_at(child_id).previous_sibling_id = previous_sibling_id; + + if (element_at(parent_id).first_child_id == null_element_id) { + element_at(parent_id).first_child_id = child_id; + } else { + element_at(previous_sibling_id).next_sibling_id = child_id; + } + element_at(parent_id).last_child_id = child_id; +} + +void ElementRegistry::check_element_id(const ElementIdentifier id) const { + if (id == null_element_id) { + throw std::out_of_range("ElementRegistry::check_id: null identifier"); + } + if (id - 1 >= m_elements.size()) { + throw std::out_of_range( + "ElementRegistry::check_id: identifier out of range"); + } +} + +void ElementRegistry::check_text_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_texts.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.hpp b/src/odr/internal/iwork/iwork_element_registry.hpp new file mode 100644 index 00000000..a7a7bbaa --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace odr::internal::iwork { + +class ElementRegistry final { +public: + struct Element final { + ElementIdentifier parent_id{null_element_id}; + ElementIdentifier first_child_id{null_element_id}; + ElementIdentifier last_child_id{null_element_id}; + ElementIdentifier previous_sibling_id{null_element_id}; + ElementIdentifier next_sibling_id{null_element_id}; + ElementType type{ElementType::none}; + }; + + struct Text final { + std::string text; + }; + + void clear() noexcept; + + [[nodiscard]] std::size_t size() const noexcept; + + std::tuple create_element(ElementType type); + std::tuple create_text_element(); + + [[nodiscard]] Element &element_at(ElementIdentifier id); + [[nodiscard]] Text &text_element_at(ElementIdentifier id); + + [[nodiscard]] const Element &element_at(ElementIdentifier id) const; + [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; + + void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + +private: + std::vector m_elements; + std::unordered_map m_texts; + + void check_element_id(ElementIdentifier id) const; + void check_text_id(ElementIdentifier id) const; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_file.cpp b/src/odr/internal/iwork/iwork_file.cpp new file mode 100644 index 00000000..22bcdfab --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// The type of the root archive says which app wrote the package. Only +/// `.pages` is pinned — a `.numbers` or `.key` fixture would be needed to read +/// theirs off, and the extension is not an answer. +FileType file_type_by_archive_type(const std::uint32_t type) { + switch (type) { + case iwork::archive_type::pages_document: + return FileType::iwork_pages; + default: + return FileType::unknown; + } +} + +/// Reads the root archive of the package's `Document` component. The component +/// list in `Index/Metadata.iwa` is not consulted: this runs on every zip a +/// caller opens, and the `Document` component is the one whose file name never +/// carries an identifier suffix. +FileType parse_file_type(const abstract::ReadableFilesystem &filesystem) { + const std::string data = + iwork::read_iwa(filesystem, AbsPath("/Index/Document.iwa")); + const std::vector objects = iwork::read_objects(data); + if (objects.empty()) { + throw NoIworkFile(); + } + + const FileType file_type = file_type_by_archive_type(objects.front().type); + if (file_type == FileType::unknown) { + throw NoIworkFile(); + } + return file_type; +} + +} // namespace + +iwork::IworkFile::IworkFile( + std::shared_ptr filesystem) + : m_filesystem{std::move(filesystem)} { + if (!m_filesystem->is_file(AbsPath("/Index/Document.iwa"))) { + throw NoIworkFile(); + } + + m_file_meta.type = parse_file_type(*m_filesystem); + m_file_meta.mimetype = mimetype_by_file_type(m_file_meta.type); + m_file_meta.document_type = document_type_by_file_type(m_file_meta.type); +} + +std::shared_ptr iwork::IworkFile::file() const noexcept { + return {}; +} + +FileType iwork::IworkFile::file_type() const noexcept { + return m_file_meta.type; +} + +std::string_view iwork::IworkFile::mimetype() const noexcept { + return m_file_meta.mimetype; +} + +FileMeta iwork::IworkFile::file_meta() const noexcept { return m_file_meta; } + +DocumentType iwork::IworkFile::document_type() const { + return m_file_meta.document_type; +} + +bool iwork::IworkFile::is_decodable() const noexcept { return true; } + +std::shared_ptr iwork::IworkFile::document() const { + switch (file_type()) { + case FileType::iwork_pages: + return std::make_shared(m_filesystem); + default: + throw UnsupportedFileType(file_type()); + } +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_file.hpp b/src/odr/internal/iwork/iwork_file.hpp new file mode 100644 index 00000000..ce86d519 --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace odr::internal::abstract { +class Document; +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// An iWork package (`.pages`, `.numbers`, `.key`). Which app wrote it is read +/// off the root archive of `Index/Document.iwa`, not off the file name, which +/// a caller may have lost. +class IworkFile final : public abstract::DocumentFile { +public: + explicit IworkFile(std::shared_ptr filesystem); + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] DocumentType document_type() const override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + [[nodiscard]] std::shared_ptr document() const override; + +private: + std::shared_ptr m_filesystem; + FileMeta m_file_meta; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp new file mode 100644 index 00000000..20056440 --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -0,0 +1,220 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// `U+2028 LINE SEPARATOR` — a line break inside a paragraph. +constexpr std::string_view line_separator = "\xe2\x80\xa8"; +/// `U+FFFC OBJECT REPLACEMENT CHARACTER` — where a drawable is anchored in the +/// text. Nothing reads drawables yet, so the anchor is dropped rather than +/// rendered as a glyph. +constexpr std::string_view object_replacement = "\xef\xbf\xbc"; + +/// The byte length of the UTF-8 sequence @p lead starts. +std::size_t utf8_length(const std::uint8_t lead) { + if (lead < 0x80) { + return 1; + } + if ((lead & 0xe0) == 0xc0) { + return 2; + } + if ((lead & 0xf0) == 0xe0) { + return 3; + } + if ((lead & 0xf8) == 0xf0) { + return 4; + } + throw std::runtime_error("iwork: text is not utf-8"); +} + +/// Translates the ascending UTF-16 code unit @p indices a storage's run tables +/// count in into byte offsets into @p text. +std::vector +utf16_offsets(const std::string_view text, + const std::vector &indices) { + std::vector result; + result.reserve(indices.size()); + + std::size_t offset = 0; + std::uint64_t unit = 0; + auto next = indices.begin(); + + for (;;) { + while (next != indices.end() && *next == unit) { + result.push_back(offset); + ++next; + } + if (next == indices.end()) { + return result; + } + if (offset >= text.size()) { + throw std::runtime_error("iwork: run table points past the text"); + } + + const std::size_t length = + utf8_length(static_cast(text[offset])); + if (offset + length > text.size()) { + throw std::runtime_error("iwork: text ends mid-character"); + } + offset += length; + // everything outside the basic multilingual plane is a surrogate pair + unit += length == 4 ? 2 : 1; + } +} + +/// The character index each paragraph of @p storage starts at. Paragraph +/// boundaries are the run table's rather than every `\n` in the text — the two +/// agree today, but the table is what says so. +std::vector paragraph_starts(const iwork::Message &storage) { + std::vector result; + + const std::optional table = + storage.bytes_field(iwork::text_storage::paragraph_styles); + if (!table.has_value()) { + return {0}; + } + + for (const iwork::Field &entry : + iwork::Message(*table).repeated_field(iwork::attribute_table::entries)) { + if (entry.type != iwork::WireType::length_delimited) { + throw std::runtime_error("iwork: malformed paragraph style table"); + } + const iwork::Message run(entry.bytes); + result.push_back( + run.number_field(iwork::attribute_table_entry::character_index) + .value_or(0)); + } + + // an empty document carries an empty table; either way the body starts at + // its first character + if (result.empty() || result.front() != 0) { + result.insert(result.begin(), 0); + } + return result; +} + +/// @p text without the drawable anchors it holds. +std::string without_anchors(const std::string_view text) { + std::string result; + result.reserve(text.size()); + + for (std::size_t position = 0; position < text.size();) { + const std::size_t anchor = text.find(object_replacement, position); + if (anchor == std::string_view::npos) { + result += text.substr(position); + break; + } + result += text.substr(position, anchor - position); + position = anchor + object_replacement.size(); + } + return result; +} + +/// Fills @p paragraph_id with the text of one paragraph, breaking it at the +/// line separators it holds. +void parse_paragraph(iwork::ElementRegistry ®istry, + const ElementIdentifier paragraph_id, + std::string_view content) { + const auto append_text = [&](const std::string_view part) { + std::string text = without_anchors(part); + if (text.empty()) { + return; + } + auto [text_id, element, payload] = registry.create_text_element(); + payload.text = std::move(text); + registry.append_child(paragraph_id, text_id); + }; + + for (std::size_t position = content.find(line_separator); + position != std::string_view::npos; + position = content.find(line_separator)) { + append_text(content.substr(0, position)); + + auto [break_id, element] = registry.create_element(ElementType::line_break); + registry.append_child(paragraph_id, break_id); + + content.remove_prefix(position + line_separator.size()); + } + append_text(content); +} + +} // namespace + +ElementIdentifier +iwork::parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files) { + Package package(files); + + const std::vector &objects = package.component("Document").objects(); + if (objects.empty() || objects.front().type != archive_type::pages_document) { + throw std::runtime_error("iwork: no pages document archive"); + } + + const Message document(objects.front().payload); + const std::optional body = + document.bytes_field(document_archive::body_storage); + if (!body.has_value()) { + throw std::runtime_error("iwork: document archive holds no body"); + } + const std::optional body_identifier = + Message(*body).number_field(reference::identifier); + if (!body_identifier.has_value()) { + throw std::runtime_error("iwork: body reference names no object"); + } + + const Object &body_object = package.object(*body_identifier); + if (body_object.type != archive_type::text_storage) { + throw std::runtime_error("iwork: body is not a text storage"); + } + const Message storage(body_object.payload); + + // the text arrives as a small number of large strings; the run tables index + // it as one + std::string text; + for (const Field &part : storage.repeated_field(text_storage::text)) { + if (part.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed text storage"); + } + text += part.bytes; + } + + const std::vector starts = + utf16_offsets(text, paragraph_starts(storage)); + + auto [root_id, root] = registry.create_element(ElementType::root); + + const std::string_view body_text(text); + for (std::size_t i = 0; i < starts.size(); ++i) { + const std::size_t begin = starts[i]; + const std::size_t end = i + 1 < starts.size() ? starts[i + 1] : text.size(); + + std::string_view content = body_text.substr(begin, end - begin); + // the paragraph mark belongs to the paragraph it ends, and the last + // paragraph of a body does not carry one + if (content.ends_with('\n')) { + content.remove_suffix(1); + } + + auto [paragraph_id, paragraph] = + registry.create_element(ElementType::paragraph); + registry.append_child(root_id, paragraph_id); + parse_paragraph(registry, paragraph_id, content); + } + + return root_id; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_parser.hpp b/src/odr/internal/iwork/iwork_parser.hpp new file mode 100644 index 00000000..82cbd72c --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { +class ElementRegistry; + +/// Parses the body of a `.pages` package into root → paragraph → text +/// elements. +/// \return the root element id. +ElementIdentifier parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_protobuf.cpp b/src/odr/internal/iwork/iwork_protobuf.cpp new file mode 100644 index 00000000..df203b33 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.cpp @@ -0,0 +1,129 @@ +#include + +#include + +namespace odr::internal { + +namespace { + +std::uint64_t read_fixed(const std::string_view in, std::size_t &position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: protobuf fixed field is cut off"); + } + + std::uint64_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + position += size; + return result; +} + +} // namespace + +std::uint64_t iwork::read_varint(const std::string_view in, + std::size_t &position) { + std::uint64_t result = 0; + for (std::uint32_t shift = 0; shift <= 63; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error("iwork: protobuf varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: protobuf varint does not terminate"); +} + +iwork::Message::Message(const std::string_view bytes) { + std::size_t position = 0; + + while (position < bytes.size()) { + const std::uint64_t key = read_varint(bytes, position); + const auto wire_type = static_cast(key & 0x07); + const auto number = static_cast(key >> 3); + if (number == 0) { + throw std::runtime_error("iwork: protobuf field number zero"); + } + + Field field; + field.number = number; + field.type = wire_type; + + switch (wire_type) { + case WireType::varint: + field.number_value = read_varint(bytes, position); + break; + case WireType::fixed64: + field.number_value = read_fixed(bytes, position, 8); + break; + case WireType::fixed32: + field.number_value = read_fixed(bytes, position, 4); + break; + case WireType::length_delimited: { + const std::uint64_t length = read_varint(bytes, position); + if (length > bytes.size() - position) { + throw std::runtime_error("iwork: protobuf field runs past the message"); + } + field.bytes = bytes.substr(position, length); + position += length; + } break; + case WireType::start_group: + case WireType::end_group: + throw std::runtime_error("iwork: protobuf group field"); + } + + m_fields.push_back(field); + } +} + +const std::vector &iwork::Message::fields() const noexcept { + return m_fields; +} + +std::optional +iwork::Message::field(const std::uint32_t number) const { + std::optional result; + for (const Field &field : m_fields) { + if (field.number == number) { + result = field; + } + } + return result; +} + +std::vector +iwork::Message::repeated_field(const std::uint32_t number) const { + std::vector result; + for (const Field &field : m_fields) { + if (field.number == number) { + result.push_back(field); + } + } + return result; +} + +std::optional +iwork::Message::number_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type == WireType::length_delimited) { + return {}; + } + return field->number_value; +} + +std::optional +iwork::Message::bytes_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type != WireType::length_delimited) { + return {}; + } + return field->bytes; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_protobuf.hpp b/src/odr/internal/iwork/iwork_protobuf.hpp new file mode 100644 index 00000000..85b75a82 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include + +namespace odr::internal::iwork { + +/// The protobuf wire types. Groups (3 and 4) are deprecated and never appear +/// in an iWork archive, so reading one is a parse error rather than a field to +/// skip. +enum class WireType : std::uint8_t { + varint = 0, + fixed64 = 1, + length_delimited = 2, + start_group = 3, + end_group = 4, + fixed32 = 5, +}; + +/// One field of a protobuf message. @ref number_value carries a varint or a +/// fixed-width field, @ref bytes a length-delimited one — a nested message, a +/// string or a packed repeated field. +struct Field final { + std::uint32_t number{}; + WireType type{WireType::varint}; + std::uint64_t number_value{}; + std::string_view bytes; +}; + +/// A protobuf message read by field number: there are no schemas to generate +/// accessors from, so the archives are read against hand-written ones. +/// +/// Nested messages stay as views into the buffer the message was read from, +/// which has to outlive it. +class Message final { +public: + explicit Message(std::string_view bytes); + + [[nodiscard]] const std::vector &fields() const noexcept; + + /// The last field numbered @p number, which is what protobuf makes of a + /// non-repeated field appearing more than once. + [[nodiscard]] std::optional field(std::uint32_t number) const; + [[nodiscard]] std::vector repeated_field(std::uint32_t number) const; + + [[nodiscard]] std::optional + number_field(std::uint32_t number) const; + [[nodiscard]] std::optional + bytes_field(std::uint32_t number) const; + +private: + std::vector m_fields; +}; + +/// Reads a varint at @p position and advances it past the field. Throws when +/// the varint does not terminate within ten bytes. +std::uint64_t read_varint(std::string_view in, std::size_t &position); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_snappy.cpp b/src/odr/internal/iwork/iwork_snappy.cpp new file mode 100644 index 00000000..63a8f931 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include + +namespace odr::internal { + +namespace { + +/// Reads @p size little-endian bytes as an unsigned integer. +std::uint32_t read_little_endian(const std::string_view in, + const std::size_t position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: snappy block ends mid-tag"); + } + + std::uint32_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + return result; +} + +/// Reads the block's uncompressed length and advances @p position past it. +std::uint32_t read_uncompressed_length(const std::string_view in, + std::size_t &position) { + std::uint32_t result = 0; + for (std::uint32_t shift = 0; shift <= 28; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error( + "iwork: snappy length varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: snappy length varint does not terminate"); +} + +} // namespace + +std::string iwork::snappy_decompress_block(const std::string_view compressed) { + std::size_t position = 0; + const std::uint32_t uncompressed_length = + read_uncompressed_length(compressed, position); + + std::string result; + result.reserve(uncompressed_length); + + while (position < compressed.size()) { + const auto tag = static_cast(compressed[position++]); + + if ((tag & 0x03) == 0) { + // literal: the length is in the tag, or in the bytes following it + std::size_t length = tag >> 2; + if (length >= 60) { + const std::size_t length_size = length - 59; + length = read_little_endian(compressed, position, length_size); + position += length_size; + } + ++length; + + if (position + length > compressed.size()) { + throw std::runtime_error("iwork: snappy literal runs past the block"); + } + result.append(compressed, position, length); + position += length; + continue; + } + + // copy: a length and a back reference into what has been written already + std::size_t length = 0; + std::size_t offset = 0; + if ((tag & 0x03) == 1) { + length = 4 + ((tag >> 2) & 0x07); + offset = (static_cast(tag >> 5) << 8) | + read_little_endian(compressed, position, 1); + position += 1; + } else { + const std::size_t offset_size = (tag & 0x03) == 2 ? 2 : 4; + length = (tag >> 2) + 1; + offset = read_little_endian(compressed, position, offset_size); + position += offset_size; + } + + if (offset == 0 || offset > result.size()) { + throw std::runtime_error("iwork: snappy copy points outside the block"); + } + // the copy may overlap what it writes, so it runs byte by byte + for (std::size_t i = 0, from = result.size() - offset; i < length; ++i) { + result.push_back(result[from + i]); + } + } + + if (result.size() != uncompressed_length) { + throw std::runtime_error("iwork: snappy block does not fill its length"); + } + return result; +} + +std::string iwork::iwa_decompress(const std::string_view framed) { + std::string result; + + std::size_t position = 0; + while (position < framed.size()) { + if (position + 4 > framed.size()) { + throw std::runtime_error("iwork: iwa block header is cut off"); + } + if (framed[position] != '\0') { + throw std::runtime_error("iwork: iwa block header is not zero"); + } + const std::uint32_t length = read_little_endian(framed, position + 1, 3); + position += 4; + + if (position + length > framed.size()) { + throw std::runtime_error("iwork: iwa block runs past the file"); + } + result += snappy_decompress_block(framed.substr(position, length)); + position += length; + } + + return result; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_snappy.hpp b/src/odr/internal/iwork/iwork_snappy.hpp new file mode 100644 index 00000000..158d97a5 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace odr::internal::iwork { + +/// Decompresses one Snappy block — a varint uncompressed length followed by +/// literal and copy tags. The stream framing Snappy ships with (the `sNaPpY` +/// identifier, per-chunk CRC-32C) is not involved, see @ref iwa_decompress. +std::string snappy_decompress_block(std::string_view compressed); + +/// Undoes the framing of an `.iwa`: `0x00`, a little-endian 24-bit compressed +/// length, then that many bytes of a Snappy block, repeated to the end. +/// Verified on `empty.pages Index/Document.iwa +0`. +std::string iwa_decompress(std::string_view framed); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_types.hpp b/src/odr/internal/iwork/iwork_types.hpp new file mode 100644 index 00000000..7cf0f962 --- /dev/null +++ b/src/odr/internal/iwork/iwork_types.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace odr::internal::iwork { + +/// The archive types and field numbers the engine reads. +/// +/// There is no spec and Apple has never published the `.proto` schemas, so +/// each of these is cited to the fixture it was read off rather than to a +/// section number, and holds for the iWork version that wrote it — see +/// `Metadata/BuildVersionHistory.plist`. A type id that is not here is one we +/// have not mapped, which the reader skips rather than throws on. +namespace archive_type { +/// `TP.DocumentArchive`, the root of a `.pages` package. +/// `empty.pages Index/Document.iwa` object 1 (iWork 13.2). +constexpr std::uint32_t pages_document = 10000; +/// `TSWP.StorageArchive`, a run of text with its run tables. +/// `empty.pages Index/Document.iwa` object 1732514 (iWork 13.2). +constexpr std::uint32_t text_storage = 2001; +} // namespace archive_type + +namespace document_archive { +/// The body text storage, as a `TSP.Reference`. +constexpr std::uint32_t body_storage = 4; +} // namespace document_archive + +namespace text_storage { +/// The text, in a small number of large strings. +constexpr std::uint32_t text = 3; +/// The paragraph style run table: one entry per paragraph, holding the +/// character index the paragraph starts at and, where it has one, its style. +constexpr std::uint32_t paragraph_styles = 5; +} // namespace text_storage + +/// A run table parallel to the text, as `TSWP.ObjectAttributeTable`. +namespace attribute_table { +constexpr std::uint32_t entries = 1; +} // namespace attribute_table + +namespace reference { +constexpr std::uint32_t identifier = 1; +} // namespace reference + +namespace attribute_table_entry { +constexpr std::uint32_t character_index = 1; +} // namespace attribute_table_entry + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 68a4e260..4a53a4ad 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,18 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoOpenDocumentFile(); } + if (as == FileType::iwork_pages) { + ODR_VERBOSE(logger, "open as iwork"); + try { + auto zip_file = std::make_unique(file); + auto filesystem = zip_file->archive()->as_filesystem(); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + throw NoIworkFile(); + } + if (as == FileType::office_open_xml_document || as == FileType::office_open_xml_presentation || as == FileType::office_open_xml_workbook || @@ -258,6 +271,13 @@ open_strategy::list_file_types(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + result.push_back(iwork::IworkFile(filesystem).file_type()); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } catch (...) { ODR_VERBOSE(logger, "failed to open as zip"); } @@ -367,6 +387,13 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "failed to open as ooxml"); } + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + return zip_file; } if (file_type == FileType::compound_file_binary_format) { @@ -545,6 +572,13 @@ open_strategy::open_document_file(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } else if (file_type == FileType::compound_file_binary_format) { ODR_VERBOSE(logger, "open as cbf"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b516952..315e50d3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,11 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_archive_test.cpp" + "src/internal/iwork/iwork_protobuf_test.cpp" + "src/internal/iwork/iwork_snappy_test.cpp" + "src/internal/iwork/pages_test.cpp" + "src/internal/odf/odf_table_test.cpp" "src/internal/oldms/doc_test.cpp" diff --git a/test/data.cmake b/test/data.cmake index ffdcf2fc..26286106 100644 --- a/test/data.cmake +++ b/test/data.cmake @@ -17,7 +17,7 @@ odr_test_data( odr_test_data( PATH "reference-output/odr-public" URL "https://github.com/opendocument-app/OpenDocument.test.output.git" - REVISION "d9cb5666399ab5873152953106dce53723fcbced") + REVISION "f8994d207ce3869e05e4ba126cd9d3de1fc73bc7") odr_test_data( PATH "reference-output/odr-private" diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp new file mode 100644 index 00000000..192e02cb --- /dev/null +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -0,0 +1,120 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string number_field(const std::uint32_t number, + const std::uint64_t value) { + return varint(number << 3) + varint(value); +} + +std::string message_field(const std::uint32_t number, + const std::string &bytes) { + return varint((number << 3) | 2) + varint(bytes.size()) + bytes; +} + +/// `TSP.ArchiveInfo`: an identifier and one `MessageInfo` per payload message. +std::string archive_info( + const std::uint64_t identifier, + const std::vector> &messages) { + std::string result = number_field(1, identifier); + for (const auto &[type, length] : messages) { + result += message_field(2, number_field(1, type) + number_field(3, length)); + } + return result; +} + +std::string +object(const std::uint64_t identifier, + const std::vector> &messages, + const std::string &payload) { + const std::string info = archive_info(identifier, messages); + return varint(info.size()) + info + payload; +} + +} // namespace + +TEST(ReadObjects, one_object) { + const std::string data = object(1, {{10000, 5}}, "hello"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].identifier, 1); + EXPECT_EQ(objects[0].type, 10000); + EXPECT_EQ(objects[0].payload, "hello"); +} + +TEST(ReadObjects, objects_follow_one_another) { + const std::string data = + object(1, {{10000, 5}}, "hello") + object(1732514, {{2001, 5}}, "world"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[1].identifier, 1732514); + EXPECT_EQ(objects[1].type, 2001); + EXPECT_EQ(objects[1].payload, "world"); +} + +// An object may hold more than one message; only the first is modelled, and +// the length of the rest is what keeps the reader in step. +TEST(ReadObjects, later_messages_are_skipped) { + const std::string data = object(1732594, {{6247, 3}, {6247, 3}}, "onetwo") + + object(2, {{222, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[0].type, 6247); + EXPECT_EQ(objects[0].payload, "one"); + EXPECT_EQ(objects[1].identifier, 2); +} + +// There is no spec and no schema registry, so a type we have not mapped is an +// app version we have not seen — the reader keeps it and moves on. +TEST(ReadObjects, unknown_type_is_kept) { + const std::string data = object(7, {{123456, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].type, 123456); +} + +TEST(ReadObjects, empty_payload) { + const std::string data = object(1732550, {{3047, 0}}, ""); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_TRUE(objects[0].payload.empty()); +} + +TEST(ReadObjects, empty_component) { EXPECT_TRUE(read_objects("").empty()); } + +TEST(ReadObjects, archive_info_runs_past_the_component) { + const std::string data = object(1, {{10000, 5}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data.substr(0, 4))); +} + +TEST(ReadObjects, payload_runs_past_the_component) { + const std::string data = object(1, {{10000, 500}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data)); +} diff --git a/test/src/internal/iwork/iwork_protobuf_test.cpp b/test/src/internal/iwork/iwork_protobuf_test.cpp new file mode 100644 index 00000000..6cc44acc --- /dev/null +++ b/test/src/internal/iwork/iwork_protobuf_test.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string key(const std::uint32_t number, const WireType type) { + return varint((number << 3) | static_cast(type)); +} + +std::string length_delimited(const std::uint32_t number, + const std::string &bytes) { + return key(number, WireType::length_delimited) + varint(bytes.size()) + bytes; +} + +void parse(const std::string &data) { + const Message message(data); + (void)message; +} + +} // namespace + +TEST(ProtobufMessage, varint_field) { + const std::string data = key(1, WireType::varint) + varint(10000); + const Message message(data); + EXPECT_EQ(message.number_field(1), 10000); +} + +// The wire format's largest varint is ten bytes. +TEST(ProtobufMessage, largest_varint) { + const std::string data = key(1, WireType::varint) + + varint(std::numeric_limits::max()); + const Message message(data); + EXPECT_EQ(message.number_field(1), std::numeric_limits::max()); +} + +TEST(ProtobufMessage, varint_does_not_terminate) { + EXPECT_ANY_THROW(parse(key(1, WireType::varint) + std::string(11, '\xff'))); +} + +TEST(ProtobufMessage, fixed_fields) { + const std::string data = + key(1, WireType::fixed32) + std::string{'\x04', '\x03', '\x02', '\x01'} + + key(2, WireType::fixed64) + std::string{'\x08', '\x07', '\x06', '\x05', + '\x04', '\x03', '\x02', '\x01'}; + const Message message(data); + EXPECT_EQ(message.number_field(1), 0x01020304); + EXPECT_EQ(message.number_field(2), 0x0102030405060708); +} + +TEST(ProtobufMessage, bytes_field) { + const std::string data = length_delimited(3, "Table of Contents"); + const Message message(data); + EXPECT_EQ(message.bytes_field(3), "Table of Contents"); +} + +TEST(ProtobufMessage, nested_message) { + const std::string data = + length_delimited(2, key(1, WireType::varint) + varint(1732588)); + const Message message(data); + const Message nested(message.bytes_field(2).value()); + EXPECT_EQ(nested.number_field(1), 1732588); +} + +TEST(ProtobufMessage, repeated_field) { + const std::string data = length_delimited(3, "a") + length_delimited(4, "b") + + length_delimited(3, "c"); + const Message message(data); + const std::vector repeated = message.repeated_field(3); + ASSERT_EQ(repeated.size(), 2); + EXPECT_EQ(repeated[0].bytes, "a"); + EXPECT_EQ(repeated[1].bytes, "c"); +} + +// A field we have no accessor for is read like any other and left alone. +TEST(ProtobufMessage, unknown_field_is_kept) { + const std::string data = + key(999, WireType::varint) + varint(1) + length_delimited(3, "text"); + const Message message(data); + EXPECT_EQ(message.fields().size(), 2); + EXPECT_EQ(message.bytes_field(3), "text"); + EXPECT_FALSE(message.bytes_field(999).has_value()); + EXPECT_FALSE(message.number_field(3).has_value()); +} + +TEST(ProtobufMessage, absent_field) { + const std::string data = length_delimited(3, "text"); + const Message message(data); + EXPECT_FALSE(message.field(4).has_value()); + EXPECT_TRUE(message.repeated_field(4).empty()); +} + +TEST(ProtobufMessage, field_runs_past_the_message) { + EXPECT_ANY_THROW( + parse(key(3, WireType::length_delimited) + varint(10) + "short")); +} + +// Groups are deprecated and no iWork archive carries one, so reading one means +// the parse went wrong rather than that a field needs skipping. +TEST(ProtobufMessage, group_field) { + EXPECT_ANY_THROW(parse(key(1, WireType::start_group))); +} + +TEST(ProtobufMessage, field_number_zero) { + EXPECT_ANY_THROW(parse(key(0, WireType::varint) + varint(1))); +} + +TEST(ReadVarint, advances_past_the_field) { + const std::string data = varint(300) + varint(1); + std::size_t position = 0; + EXPECT_EQ(read_varint(data, position), 300); + EXPECT_EQ(position, 2); + EXPECT_EQ(read_varint(data, position), 1); + EXPECT_EQ(position, 3); +} diff --git a/test/src/internal/iwork/iwork_snappy_test.cpp b/test/src/internal/iwork/iwork_snappy_test.cpp new file mode 100644 index 00000000..1ceaf495 --- /dev/null +++ b/test/src/internal/iwork/iwork_snappy_test.cpp @@ -0,0 +1,122 @@ +#include + +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +/// A Snappy block: the uncompressed length as a varint, then @p body. +std::string block(const std::size_t uncompressed_length, + const std::string &body) { + std::string result; + for (std::size_t rest = uncompressed_length;;) { + const auto byte = static_cast(rest & 0x7f); + rest >>= 7; + result.push_back(rest == 0 ? byte : static_cast(byte | 0x80)); + if (rest == 0) { + break; + } + } + return result + body; +} + +/// A literal tag for @p text, in the form that carries the length inline. +std::string literal(const std::string &text) { + return std::string(1, static_cast((text.size() - 1) << 2)) + text; +} + +/// One `.iwa` block header plus @p body. +std::string framed(const std::string &body) { + const std::size_t length = body.size(); + const std::string header{'\0', static_cast(length & 0xff), + static_cast((length >> 8) & 0xff), + static_cast((length >> 16) & 0xff)}; + return header + body; +} + +} // namespace + +TEST(SnappyDecompressBlock, literal) { + EXPECT_EQ(snappy_decompress_block(block(5, literal("hello"))), "hello"); +} + +TEST(SnappyDecompressBlock, empty) { + EXPECT_EQ(snappy_decompress_block(block(0, "")), ""); +} + +// A literal of 61 bytes or more names its length in the bytes after the tag. +TEST(SnappyDecompressBlock, long_literal) { + const std::string text(300, 'x'); + const std::string body = std::string{'\xf4', '\x2b', '\x01'} + text; + EXPECT_EQ(snappy_decompress_block(block(text.size(), body)), text); +} + +// Copy tag 1: a three-bit length and a ten-bit offset. +TEST(SnappyDecompressBlock, copy_with_one_byte_offset) { + const std::string body = literal("abc") + std::string{'\x09', '\x03'}; + EXPECT_EQ(snappy_decompress_block(block(9, body)), "abcabcabc"); +} + +// Copy tag 2: a six-bit length and a two-byte offset. +TEST(SnappyDecompressBlock, copy_with_two_byte_offset) { + const std::string body = + literal("abcd") + std::string{'\x0e', '\x04', '\x00'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abcdabcd"); +} + +// The run a copy reads may be the one it is writing. +TEST(SnappyDecompressBlock, overlapping_copy) { + const std::string body = literal("ab") + std::string{'\x09', '\x02'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abababab"); +} + +TEST(SnappyDecompressBlock, length_does_not_match) { + EXPECT_ANY_THROW(std::ignore = + snappy_decompress_block(block(6, literal("hello")))); +} + +TEST(SnappyDecompressBlock, literal_runs_past_the_block) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block( + block(5, std::string{'\x10'} + "hel"))); +} + +TEST(SnappyDecompressBlock, copy_points_outside_the_block) { + const std::string body = literal("abc") + std::string{'\x09', '\x09'}; + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(block(9, body))); +} + +TEST(SnappyDecompressBlock, length_varint_does_not_terminate) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block("\x80\x80\x80")); +} + +TEST(IwaDecompress, one_block) { + EXPECT_EQ(iwa_decompress(framed(block(5, literal("hello")))), "hello"); +} + +// A file is as many blocks as it takes; they concatenate. +TEST(IwaDecompress, two_blocks) { + const std::string data = + framed(block(5, literal("hello"))) + framed(block(6, literal(" world"))); + EXPECT_EQ(iwa_decompress(data), "hello world"); +} + +TEST(IwaDecompress, empty_file) { EXPECT_EQ(iwa_decompress(""), ""); } + +TEST(IwaDecompress, header_is_not_zero) { + std::string data = framed(block(5, literal("hello"))); + data[0] = '\x01'; + EXPECT_ANY_THROW(std::ignore = iwa_decompress(data)); +} + +TEST(IwaDecompress, truncated_mid_block) { + const std::string data = framed(block(5, literal("hello"))); + EXPECT_ANY_THROW(std::ignore = + iwa_decompress(data.substr(0, data.size() - 2))); +} + +TEST(IwaDecompress, truncated_header) { + EXPECT_ANY_THROW(std::ignore = iwa_decompress(std::string{'\0', '\x07'})); +} diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp new file mode 100644 index 00000000..820dfc95 --- /dev/null +++ b/test/src/internal/iwork/pages_test.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +using namespace odr; +using odr::test::TestData; + +namespace { + +/// The paragraphs of a text root, a line break reading as a newline. +std::vector paragraphs(const Element root) { + std::vector result; + + for (const Element paragraph : root.children()) { + EXPECT_EQ(paragraph.type(), ElementType::paragraph); + + std::string text; + for (const Element child : paragraph.children()) { + if (child.type() == ElementType::line_break) { + text += '\n'; + } else { + text += child.as_text().content(); + } + } + result.push_back(std::move(text)); + } + + return result; +} + +} // namespace + +TEST(Iwork, pages_is_detected_by_content) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + const std::string path = + TestData::test_file_path("odr-public/pages/style-various-1.pages"); + + EXPECT_THAT(list_file_types(path, logger), + testing::Contains(FileType::iwork_pages)); + + const DecodedFile file(path, logger); + EXPECT_EQ(file.file_type(), FileType::iwork_pages); + EXPECT_EQ(file.file_category(), FileCategory::document); + EXPECT_EQ(file.as_document_file().document_type(), DocumentType::text); +} + +// A document with nothing in it must come back with an empty body rather than +// throw: `empty.pages` carries a body storage that holds no text at all. +TEST(Iwork, pages_empty) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/empty.pages"), logger); + EXPECT_EQ(document_file.file_type(), FileType::iwork_pages); + + const Document document = document_file.document(); + EXPECT_EQ(document.document_type(), DocumentType::text); + EXPECT_FALSE(document.is_editable()); + EXPECT_FALSE(document.is_savable(false)); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{""})); +} + +TEST(Iwork, pages_body_text) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/style-various-1.pages"), + logger); + + const Document document = document_file.document(); + const std::vector text = paragraphs(document.root_element()); + + // one element per paragraph of the body, including the empty ones that + // separate its sections + ASSERT_EQ(text.size(), 54); + EXPECT_EQ(text[0], "Table of Contents"); + // the anchor of a drawable is dropped: nothing reads drawables yet + EXPECT_EQ(text[1], ""); + EXPECT_EQ(text[4], "Headline"); + EXPECT_EQ(text[5], "Nested Headline"); + EXPECT_EQ(text[7], "Text"); + EXPECT_EQ(text[9], "Hyperlink google"); + EXPECT_EQ(text[11], "Default"); + EXPECT_EQ(text[12], "Bold"); + EXPECT_EQ(text.back(), "image"); +} + +// Components share names — `style-various-1.pages` holds two dozen called +// `Tables/DataList` — so the package has to load them by locator. Keying on +// the name hands back the wrong file and leaves the rest never loaded, which +// shows up as an object nothing can resolve. +TEST(Iwork, package_resolves_across_components) { + using odr::internal::iwork::Package; + + const auto file = + std::make_shared(odr::internal::AbsPath( + TestData::test_file_path("odr-public/pages/style-various-1.pages"))); + const auto filesystem = + odr::internal::zip::ZipFile(file).archive()->as_filesystem(); + + Package package(*filesystem); + + EXPECT_EQ(package.component("Document").objects().front().identifier, 1); + // the stylesheet, which is a component of its own + EXPECT_EQ(package.object(1732588).identifier, 1732588); + // the root of a `Tables/DataList` that is not the first one of that name + EXPECT_EQ(package.object(1732940).identifier, 1732940); +} diff --git a/test/src/odr_test.cpp b/test/src/odr_test.cpp index 11bd2d6e..1f7a5fd2 100644 --- a/test/src/odr_test.cpp +++ b/test/src/odr_test.cpp @@ -27,7 +27,7 @@ namespace { std::vector every_file_type() { std::vector result; for (auto i = static_cast(FileType::unknown); - i <= static_cast(FileType::xml); ++i) { + i <= static_cast(FileType::iwork_keynote); ++i) { result.push_back(static_cast(i)); } return result;