From 6fbdb0ec14b201fabbc0efd22ecfb57b4d9368ad Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:13 +0200 Subject: [PATCH 1/7] feat(iwork): unpack the snappy framing of an apple `.iwa` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The framing an iWork package uses is Apple's own — a four-byte header per block, `0x00` and a little-endian 24-bit compressed length — so stock Snappy stream decoding does not apply and only the block decoder does. That is a varint length plus literal and copy tags, which is less code than a dependency would be and keeps it out of the wasm, android and apple builds. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 2 + src/odr/internal/iwork/iwork_snappy.cpp | 130 ++++++++++++++++++ src/odr/internal/iwork/iwork_snappy.hpp | 18 +++ test/CMakeLists.txt | 2 + test/src/internal/iwork/iwork_snappy_test.cpp | 122 ++++++++++++++++ 5 files changed, 274 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_snappy.cpp create mode 100644 src/odr/internal/iwork/iwork_snappy.hpp create mode 100644 test/src/internal/iwork/iwork_snappy_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 04feb219..f58ca415 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,8 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_snappy.cpp" + "src/odr/internal/json/json_file.cpp" "src/odr/internal/json/json_util.cpp" diff --git a/src/odr/internal/iwork/iwork_snappy.cpp b/src/odr/internal/iwork/iwork_snappy.cpp new file mode 100644 index 00000000..63a8f931 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include + +namespace odr::internal { + +namespace { + +/// Reads @p size little-endian bytes as an unsigned integer. +std::uint32_t read_little_endian(const std::string_view in, + const std::size_t position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: snappy block ends mid-tag"); + } + + std::uint32_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + return result; +} + +/// Reads the block's uncompressed length and advances @p position past it. +std::uint32_t read_uncompressed_length(const std::string_view in, + std::size_t &position) { + std::uint32_t result = 0; + for (std::uint32_t shift = 0; shift <= 28; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error( + "iwork: snappy length varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: snappy length varint does not terminate"); +} + +} // namespace + +std::string iwork::snappy_decompress_block(const std::string_view compressed) { + std::size_t position = 0; + const std::uint32_t uncompressed_length = + read_uncompressed_length(compressed, position); + + std::string result; + result.reserve(uncompressed_length); + + while (position < compressed.size()) { + const auto tag = static_cast(compressed[position++]); + + if ((tag & 0x03) == 0) { + // literal: the length is in the tag, or in the bytes following it + std::size_t length = tag >> 2; + if (length >= 60) { + const std::size_t length_size = length - 59; + length = read_little_endian(compressed, position, length_size); + position += length_size; + } + ++length; + + if (position + length > compressed.size()) { + throw std::runtime_error("iwork: snappy literal runs past the block"); + } + result.append(compressed, position, length); + position += length; + continue; + } + + // copy: a length and a back reference into what has been written already + std::size_t length = 0; + std::size_t offset = 0; + if ((tag & 0x03) == 1) { + length = 4 + ((tag >> 2) & 0x07); + offset = (static_cast(tag >> 5) << 8) | + read_little_endian(compressed, position, 1); + position += 1; + } else { + const std::size_t offset_size = (tag & 0x03) == 2 ? 2 : 4; + length = (tag >> 2) + 1; + offset = read_little_endian(compressed, position, offset_size); + position += offset_size; + } + + if (offset == 0 || offset > result.size()) { + throw std::runtime_error("iwork: snappy copy points outside the block"); + } + // the copy may overlap what it writes, so it runs byte by byte + for (std::size_t i = 0, from = result.size() - offset; i < length; ++i) { + result.push_back(result[from + i]); + } + } + + if (result.size() != uncompressed_length) { + throw std::runtime_error("iwork: snappy block does not fill its length"); + } + return result; +} + +std::string iwork::iwa_decompress(const std::string_view framed) { + std::string result; + + std::size_t position = 0; + while (position < framed.size()) { + if (position + 4 > framed.size()) { + throw std::runtime_error("iwork: iwa block header is cut off"); + } + if (framed[position] != '\0') { + throw std::runtime_error("iwork: iwa block header is not zero"); + } + const std::uint32_t length = read_little_endian(framed, position + 1, 3); + position += 4; + + if (position + length > framed.size()) { + throw std::runtime_error("iwork: iwa block runs past the file"); + } + result += snappy_decompress_block(framed.substr(position, length)); + position += length; + } + + return result; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_snappy.hpp b/src/odr/internal/iwork/iwork_snappy.hpp new file mode 100644 index 00000000..158d97a5 --- /dev/null +++ b/src/odr/internal/iwork/iwork_snappy.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace odr::internal::iwork { + +/// Decompresses one Snappy block — a varint uncompressed length followed by +/// literal and copy tags. The stream framing Snappy ships with (the `sNaPpY` +/// identifier, per-chunk CRC-32C) is not involved, see @ref iwa_decompress. +std::string snappy_decompress_block(std::string_view compressed); + +/// Undoes the framing of an `.iwa`: `0x00`, a little-endian 24-bit compressed +/// length, then that many bytes of a Snappy block, repeated to the end. +/// Verified on `empty.pages Index/Document.iwa +0`. +std::string iwa_decompress(std::string_view framed); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b516952..0cb5c128 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,8 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_snappy_test.cpp" + "src/internal/odf/odf_table_test.cpp" "src/internal/oldms/doc_test.cpp" diff --git a/test/src/internal/iwork/iwork_snappy_test.cpp b/test/src/internal/iwork/iwork_snappy_test.cpp new file mode 100644 index 00000000..1ceaf495 --- /dev/null +++ b/test/src/internal/iwork/iwork_snappy_test.cpp @@ -0,0 +1,122 @@ +#include + +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +/// A Snappy block: the uncompressed length as a varint, then @p body. +std::string block(const std::size_t uncompressed_length, + const std::string &body) { + std::string result; + for (std::size_t rest = uncompressed_length;;) { + const auto byte = static_cast(rest & 0x7f); + rest >>= 7; + result.push_back(rest == 0 ? byte : static_cast(byte | 0x80)); + if (rest == 0) { + break; + } + } + return result + body; +} + +/// A literal tag for @p text, in the form that carries the length inline. +std::string literal(const std::string &text) { + return std::string(1, static_cast((text.size() - 1) << 2)) + text; +} + +/// One `.iwa` block header plus @p body. +std::string framed(const std::string &body) { + const std::size_t length = body.size(); + const std::string header{'\0', static_cast(length & 0xff), + static_cast((length >> 8) & 0xff), + static_cast((length >> 16) & 0xff)}; + return header + body; +} + +} // namespace + +TEST(SnappyDecompressBlock, literal) { + EXPECT_EQ(snappy_decompress_block(block(5, literal("hello"))), "hello"); +} + +TEST(SnappyDecompressBlock, empty) { + EXPECT_EQ(snappy_decompress_block(block(0, "")), ""); +} + +// A literal of 61 bytes or more names its length in the bytes after the tag. +TEST(SnappyDecompressBlock, long_literal) { + const std::string text(300, 'x'); + const std::string body = std::string{'\xf4', '\x2b', '\x01'} + text; + EXPECT_EQ(snappy_decompress_block(block(text.size(), body)), text); +} + +// Copy tag 1: a three-bit length and a ten-bit offset. +TEST(SnappyDecompressBlock, copy_with_one_byte_offset) { + const std::string body = literal("abc") + std::string{'\x09', '\x03'}; + EXPECT_EQ(snappy_decompress_block(block(9, body)), "abcabcabc"); +} + +// Copy tag 2: a six-bit length and a two-byte offset. +TEST(SnappyDecompressBlock, copy_with_two_byte_offset) { + const std::string body = + literal("abcd") + std::string{'\x0e', '\x04', '\x00'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abcdabcd"); +} + +// The run a copy reads may be the one it is writing. +TEST(SnappyDecompressBlock, overlapping_copy) { + const std::string body = literal("ab") + std::string{'\x09', '\x02'}; + EXPECT_EQ(snappy_decompress_block(block(8, body)), "abababab"); +} + +TEST(SnappyDecompressBlock, length_does_not_match) { + EXPECT_ANY_THROW(std::ignore = + snappy_decompress_block(block(6, literal("hello")))); +} + +TEST(SnappyDecompressBlock, literal_runs_past_the_block) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block( + block(5, std::string{'\x10'} + "hel"))); +} + +TEST(SnappyDecompressBlock, copy_points_outside_the_block) { + const std::string body = literal("abc") + std::string{'\x09', '\x09'}; + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block(block(9, body))); +} + +TEST(SnappyDecompressBlock, length_varint_does_not_terminate) { + EXPECT_ANY_THROW(std::ignore = snappy_decompress_block("\x80\x80\x80")); +} + +TEST(IwaDecompress, one_block) { + EXPECT_EQ(iwa_decompress(framed(block(5, literal("hello")))), "hello"); +} + +// A file is as many blocks as it takes; they concatenate. +TEST(IwaDecompress, two_blocks) { + const std::string data = + framed(block(5, literal("hello"))) + framed(block(6, literal(" world"))); + EXPECT_EQ(iwa_decompress(data), "hello world"); +} + +TEST(IwaDecompress, empty_file) { EXPECT_EQ(iwa_decompress(""), ""); } + +TEST(IwaDecompress, header_is_not_zero) { + std::string data = framed(block(5, literal("hello"))); + data[0] = '\x01'; + EXPECT_ANY_THROW(std::ignore = iwa_decompress(data)); +} + +TEST(IwaDecompress, truncated_mid_block) { + const std::string data = framed(block(5, literal("hello"))); + EXPECT_ANY_THROW(std::ignore = + iwa_decompress(data.substr(0, data.size() - 2))); +} + +TEST(IwaDecompress, truncated_header) { + EXPECT_ANY_THROW(std::ignore = iwa_decompress(std::string{'\0', '\x07'})); +} From afa207928e7a18ce4a16e796ed8e5586c232580e Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:23 +0200 Subject: [PATCH 2/7] feat(iwork): read the protobuf wire format An iWork archive is protobuf, but Apple has never published the `.proto` schemas, so there is nothing for a code generator to generate and linking conan `protobuf` would drag it into every downstream build to replace this. Only the wire format is needed: varints, the three fixed and length-delimited forms, and unknown fields carried along rather than dropped. A group means the parse went wrong, so it throws. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 1 + src/odr/internal/iwork/iwork_protobuf.cpp | 129 +++++++++++++++++ src/odr/internal/iwork/iwork_protobuf.hpp | 61 ++++++++ test/CMakeLists.txt | 1 + .../internal/iwork/iwork_protobuf_test.cpp | 134 ++++++++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_protobuf.cpp create mode 100644 src/odr/internal/iwork/iwork_protobuf.hpp create mode 100644 test/src/internal/iwork/iwork_protobuf_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f58ca415..6fa2cd08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,7 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" "src/odr/internal/json/json_file.cpp" diff --git a/src/odr/internal/iwork/iwork_protobuf.cpp b/src/odr/internal/iwork/iwork_protobuf.cpp new file mode 100644 index 00000000..df203b33 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.cpp @@ -0,0 +1,129 @@ +#include + +#include + +namespace odr::internal { + +namespace { + +std::uint64_t read_fixed(const std::string_view in, std::size_t &position, + const std::size_t size) { + if (position + size > in.size()) { + throw std::runtime_error("iwork: protobuf fixed field is cut off"); + } + + std::uint64_t result = 0; + for (std::size_t i = 0; i < size; ++i) { + result |= + static_cast(static_cast(in[position + i])) + << (8 * i); + } + position += size; + return result; +} + +} // namespace + +std::uint64_t iwork::read_varint(const std::string_view in, + std::size_t &position) { + std::uint64_t result = 0; + for (std::uint32_t shift = 0; shift <= 63; shift += 7) { + if (position >= in.size()) { + throw std::runtime_error("iwork: protobuf varint does not terminate"); + } + const auto byte = static_cast(in[position++]); + result |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return result; + } + } + throw std::runtime_error("iwork: protobuf varint does not terminate"); +} + +iwork::Message::Message(const std::string_view bytes) { + std::size_t position = 0; + + while (position < bytes.size()) { + const std::uint64_t key = read_varint(bytes, position); + const auto wire_type = static_cast(key & 0x07); + const auto number = static_cast(key >> 3); + if (number == 0) { + throw std::runtime_error("iwork: protobuf field number zero"); + } + + Field field; + field.number = number; + field.type = wire_type; + + switch (wire_type) { + case WireType::varint: + field.number_value = read_varint(bytes, position); + break; + case WireType::fixed64: + field.number_value = read_fixed(bytes, position, 8); + break; + case WireType::fixed32: + field.number_value = read_fixed(bytes, position, 4); + break; + case WireType::length_delimited: { + const std::uint64_t length = read_varint(bytes, position); + if (length > bytes.size() - position) { + throw std::runtime_error("iwork: protobuf field runs past the message"); + } + field.bytes = bytes.substr(position, length); + position += length; + } break; + case WireType::start_group: + case WireType::end_group: + throw std::runtime_error("iwork: protobuf group field"); + } + + m_fields.push_back(field); + } +} + +const std::vector &iwork::Message::fields() const noexcept { + return m_fields; +} + +std::optional +iwork::Message::field(const std::uint32_t number) const { + std::optional result; + for (const Field &field : m_fields) { + if (field.number == number) { + result = field; + } + } + return result; +} + +std::vector +iwork::Message::repeated_field(const std::uint32_t number) const { + std::vector result; + for (const Field &field : m_fields) { + if (field.number == number) { + result.push_back(field); + } + } + return result; +} + +std::optional +iwork::Message::number_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type == WireType::length_delimited) { + return {}; + } + return field->number_value; +} + +std::optional +iwork::Message::bytes_field(const std::uint32_t number) const { + const std::optional field = this->field(number); + if (!field.has_value() || field->type != WireType::length_delimited) { + return {}; + } + return field->bytes; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_protobuf.hpp b/src/odr/internal/iwork/iwork_protobuf.hpp new file mode 100644 index 00000000..85b75a82 --- /dev/null +++ b/src/odr/internal/iwork/iwork_protobuf.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include + +namespace odr::internal::iwork { + +/// The protobuf wire types. Groups (3 and 4) are deprecated and never appear +/// in an iWork archive, so reading one is a parse error rather than a field to +/// skip. +enum class WireType : std::uint8_t { + varint = 0, + fixed64 = 1, + length_delimited = 2, + start_group = 3, + end_group = 4, + fixed32 = 5, +}; + +/// One field of a protobuf message. @ref number_value carries a varint or a +/// fixed-width field, @ref bytes a length-delimited one — a nested message, a +/// string or a packed repeated field. +struct Field final { + std::uint32_t number{}; + WireType type{WireType::varint}; + std::uint64_t number_value{}; + std::string_view bytes; +}; + +/// A protobuf message read by field number: there are no schemas to generate +/// accessors from, so the archives are read against hand-written ones. +/// +/// Nested messages stay as views into the buffer the message was read from, +/// which has to outlive it. +class Message final { +public: + explicit Message(std::string_view bytes); + + [[nodiscard]] const std::vector &fields() const noexcept; + + /// The last field numbered @p number, which is what protobuf makes of a + /// non-repeated field appearing more than once. + [[nodiscard]] std::optional field(std::uint32_t number) const; + [[nodiscard]] std::vector repeated_field(std::uint32_t number) const; + + [[nodiscard]] std::optional + number_field(std::uint32_t number) const; + [[nodiscard]] std::optional + bytes_field(std::uint32_t number) const; + +private: + std::vector m_fields; +}; + +/// Reads a varint at @p position and advances it past the field. Throws when +/// the varint does not terminate within ten bytes. +std::uint64_t read_varint(std::string_view in, std::size_t &position); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0cb5c128..0161fc4b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" "src/internal/odf/odf_table_test.cpp" diff --git a/test/src/internal/iwork/iwork_protobuf_test.cpp b/test/src/internal/iwork/iwork_protobuf_test.cpp new file mode 100644 index 00000000..6cc44acc --- /dev/null +++ b/test/src/internal/iwork/iwork_protobuf_test.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string key(const std::uint32_t number, const WireType type) { + return varint((number << 3) | static_cast(type)); +} + +std::string length_delimited(const std::uint32_t number, + const std::string &bytes) { + return key(number, WireType::length_delimited) + varint(bytes.size()) + bytes; +} + +void parse(const std::string &data) { + const Message message(data); + (void)message; +} + +} // namespace + +TEST(ProtobufMessage, varint_field) { + const std::string data = key(1, WireType::varint) + varint(10000); + const Message message(data); + EXPECT_EQ(message.number_field(1), 10000); +} + +// The wire format's largest varint is ten bytes. +TEST(ProtobufMessage, largest_varint) { + const std::string data = key(1, WireType::varint) + + varint(std::numeric_limits::max()); + const Message message(data); + EXPECT_EQ(message.number_field(1), std::numeric_limits::max()); +} + +TEST(ProtobufMessage, varint_does_not_terminate) { + EXPECT_ANY_THROW(parse(key(1, WireType::varint) + std::string(11, '\xff'))); +} + +TEST(ProtobufMessage, fixed_fields) { + const std::string data = + key(1, WireType::fixed32) + std::string{'\x04', '\x03', '\x02', '\x01'} + + key(2, WireType::fixed64) + std::string{'\x08', '\x07', '\x06', '\x05', + '\x04', '\x03', '\x02', '\x01'}; + const Message message(data); + EXPECT_EQ(message.number_field(1), 0x01020304); + EXPECT_EQ(message.number_field(2), 0x0102030405060708); +} + +TEST(ProtobufMessage, bytes_field) { + const std::string data = length_delimited(3, "Table of Contents"); + const Message message(data); + EXPECT_EQ(message.bytes_field(3), "Table of Contents"); +} + +TEST(ProtobufMessage, nested_message) { + const std::string data = + length_delimited(2, key(1, WireType::varint) + varint(1732588)); + const Message message(data); + const Message nested(message.bytes_field(2).value()); + EXPECT_EQ(nested.number_field(1), 1732588); +} + +TEST(ProtobufMessage, repeated_field) { + const std::string data = length_delimited(3, "a") + length_delimited(4, "b") + + length_delimited(3, "c"); + const Message message(data); + const std::vector repeated = message.repeated_field(3); + ASSERT_EQ(repeated.size(), 2); + EXPECT_EQ(repeated[0].bytes, "a"); + EXPECT_EQ(repeated[1].bytes, "c"); +} + +// A field we have no accessor for is read like any other and left alone. +TEST(ProtobufMessage, unknown_field_is_kept) { + const std::string data = + key(999, WireType::varint) + varint(1) + length_delimited(3, "text"); + const Message message(data); + EXPECT_EQ(message.fields().size(), 2); + EXPECT_EQ(message.bytes_field(3), "text"); + EXPECT_FALSE(message.bytes_field(999).has_value()); + EXPECT_FALSE(message.number_field(3).has_value()); +} + +TEST(ProtobufMessage, absent_field) { + const std::string data = length_delimited(3, "text"); + const Message message(data); + EXPECT_FALSE(message.field(4).has_value()); + EXPECT_TRUE(message.repeated_field(4).empty()); +} + +TEST(ProtobufMessage, field_runs_past_the_message) { + EXPECT_ANY_THROW( + parse(key(3, WireType::length_delimited) + varint(10) + "short")); +} + +// Groups are deprecated and no iWork archive carries one, so reading one means +// the parse went wrong rather than that a field needs skipping. +TEST(ProtobufMessage, group_field) { + EXPECT_ANY_THROW(parse(key(1, WireType::start_group))); +} + +TEST(ProtobufMessage, field_number_zero) { + EXPECT_ANY_THROW(parse(key(0, WireType::varint) + varint(1))); +} + +TEST(ReadVarint, advances_past_the_field) { + const std::string data = varint(300) + varint(1); + std::size_t position = 0; + EXPECT_EQ(read_varint(data, position), 300); + EXPECT_EQ(position, 2); + EXPECT_EQ(read_varint(data, position), 1); + EXPECT_EQ(position, 3); +} From e42219fdcccc969d098fdf071640016e71c4a437 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:53:33 +0200 Subject: [PATCH 3/7] feat(iwork): index the objects an iwork package holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An `.iwa` is not a tree but a sequence of archived objects, each a `TSP.ArchiveInfo` naming an identifier and the type of the messages that follow. Objects reference each other by identifier across components, so the package reads its component list from `Index/Metadata.iwa` first — the file names carry identifier suffixes often enough that globbing for them finds nothing — and decompresses a component when something in it is asked for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 1 + src/odr/internal/iwork/iwork_archive.cpp | 180 ++++++++++++++++++ src/odr/internal/iwork/iwork_archive.hpp | 97 ++++++++++ test/CMakeLists.txt | 1 + .../src/internal/iwork/iwork_archive_test.cpp | 120 ++++++++++++ 5 files changed, 399 insertions(+) create mode 100644 src/odr/internal/iwork/iwork_archive.cpp create mode 100644 src/odr/internal/iwork/iwork_archive.hpp create mode 100644 test/src/internal/iwork/iwork_archive_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fa2cd08..fed40439 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,7 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/text_file.cpp" "src/odr/internal/html/xml_file.cpp" + "src/odr/internal/iwork/iwork_archive.cpp" "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp new file mode 100644 index 00000000..487ddcd1 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -0,0 +1,180 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// Field numbers of `TSP.PackageMetadata` and the `ComponentInfo` it repeats, +/// read off `empty.pages Index/Metadata.iwa` (object 2, type 11006). +constexpr std::uint32_t package_metadata_components = 3; +constexpr std::uint32_t component_info_identifier = 1; +constexpr std::uint32_t component_info_preferred_locator = 2; +constexpr std::uint32_t component_info_locator = 3; + +/// Field numbers of `TSP.ArchiveInfo` and the `MessageInfo` it repeats. +constexpr std::uint32_t archive_info_identifier = 1; +constexpr std::uint32_t archive_info_messages = 2; +constexpr std::uint32_t message_info_type = 1; +constexpr std::uint32_t message_info_length = 3; + +AbsPath component_path(const std::string &locator) { + return AbsPath("/Index").join(RelPath(locator + ".iwa")); +} + +} // namespace + +std::string iwork::read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path) { + const std::shared_ptr file = filesystem.open(path); + if (!file) { + throw std::runtime_error("iwork: missing " + path.string()); + } + const std::unique_ptr stream = file->stream(); + return iwa_decompress(util::stream::read(*stream)); +} + +std::vector iwork::read_objects(const std::string_view data) { + std::vector result; + + std::size_t position = 0; + while (position < data.size()) { + const std::uint64_t info_length = read_varint(data, position); + if (info_length > data.size() - position) { + throw std::runtime_error("iwork: archive info runs past the component"); + } + const Message info(data.substr(position, info_length)); + position += info_length; + + Object object; + object.identifier = info.number_field(archive_info_identifier).value_or(0); + + // the payload holds every message the info names, back to back; only the + // first is modelled, the length of the rest is what skips them + std::size_t payload_length = 0; + std::size_t first_length = 0; + bool first = true; + for (const Field &message : info.repeated_field(archive_info_messages)) { + if (message.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed message info"); + } + const Message message_info(message.bytes); + const std::uint64_t length = + message_info.number_field(message_info_length).value_or(0); + if (first) { + object.type = static_cast( + message_info.number_field(message_info_type).value_or(0)); + first_length = length; + first = false; + } + payload_length += length; + } + + if (payload_length > data.size() - position) { + throw std::runtime_error("iwork: object payload runs past the component"); + } + object.payload = data.substr(position, first_length); + position += payload_length; + + result.push_back(object); + } + + return result; +} + +iwork::Component::Component(std::string locator, std::string data) + : m_locator{std::move(locator)}, + m_data{std::make_unique(std::move(data))}, + m_objects{read_objects(*m_data)} {} + +const std::string &iwork::Component::locator() const noexcept { + return m_locator; +} + +const std::vector &iwork::Component::objects() const noexcept { + return m_objects; +} + +iwork::Package::Package(const abstract::ReadableFilesystem &filesystem) + : m_filesystem{&filesystem} { + const std::string data = read_iwa(filesystem, AbsPath("/Index/Metadata.iwa")); + const std::vector objects = read_objects(data); + if (objects.empty()) { + throw std::runtime_error("iwork: empty package metadata"); + } + + const Message metadata(objects.front().payload); + for (const Field &component : + metadata.repeated_field(package_metadata_components)) { + if (component.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed component info"); + } + const Message info(component.bytes); + + ComponentInfo result; + result.identifier = + info.number_field(component_info_identifier).value_or(0); + result.name = std::string(info.bytes_field(component_info_preferred_locator) + .value_or(std::string_view())); + result.locator = std::string( + info.bytes_field(component_info_locator).value_or(result.name)); + if (result.name.empty()) { + throw std::runtime_error("iwork: component without a name"); + } + m_component_infos.push_back(std::move(result)); + } +} + +const iwork::Component &iwork::Package::component(const std::string &name) { + const auto it = + std::ranges::find(m_component_infos, name, &ComponentInfo::name); + if (it == std::ranges::end(m_component_infos)) { + throw std::runtime_error("iwork: no component named " + name); + } + return load_(*it); +} + +const iwork::Object &iwork::Package::object(const std::uint64_t identifier) { + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + + for (const ComponentInfo &info : m_component_infos) { + load_(info); + if (const auto it = m_objects.find(identifier); it != m_objects.end()) { + return *it->second; + } + } + + throw std::runtime_error("iwork: no object " + std::to_string(identifier)); +} + +const iwork::Component &iwork::Package::load_(const ComponentInfo &info) { + // by locator, not by name: a name is shared across components, so keying on + // it would hand back the wrong file and leave the other never loaded + if (const auto it = + std::ranges::find(m_components, info.locator, &Component::locator); + it != std::ranges::end(m_components)) { + return *it; + } + + const Component &component = m_components.emplace_back( + info.locator, read_iwa(*m_filesystem, component_path(info.locator))); + for (const Object &object : component.objects()) { + m_objects.emplace(object.identifier, &object); + } + return component; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp new file mode 100644 index 00000000..7a286e01 --- /dev/null +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal { +class AbsPath; +} // namespace odr::internal + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// One archived object. `TSP.ArchiveInfo` names its identifier (field 1) and, +/// per payload message, a `MessageInfo` (field 2) carrying the message type +/// and its length. An object usually holds one message; where it holds more, +/// only the first is modelled. +/// +/// Verified on `empty.pages Index/Document.iwa +0`: `08 01` (identifier 1), +/// `12 52` (an 82-byte `MessageInfo`), `08 90 4e` (type 10000), `18 e0 0c` +/// (payload length 1632). +struct Object final { + std::uint64_t identifier{}; + std::uint32_t type{}; + std::string_view payload; +}; + +/// The objects of one `.iwa`, over the bytes it decompressed to. Object +/// payloads are views into those bytes. +class Component final { +public: + Component(std::string locator, std::string data); + + /// The file the component was loaded from, without `/Index/` and `.iwa`. + /// Unlike its name, this is unique across the package. + [[nodiscard]] const std::string &locator() const noexcept; + [[nodiscard]] const std::vector &objects() const noexcept; + +private: + std::string m_locator; + std::unique_ptr m_data; + std::vector m_objects; +}; + +/// An iWork package: the component list from `Index/Metadata.iwa`, and the +/// components loaded from it so far. +/// +/// Objects reference each other by identifier across components, so the list +/// is read first and a component is decompressed when something in it is +/// asked for. +class Package final { +public: + explicit Package(const abstract::ReadableFilesystem &filesystem); + + /// The first component named @p name in the package's component list — a + /// name is not unique, `Tables/DataList` names dozens. Throws when the + /// package holds none. + const Component &component(const std::string &name); + + /// The object @p identifier names, loading components until it is found. + /// Throws when no component holds it. + const Object &object(std::uint64_t identifier); + +private: + /// One entry of `TSP.PackageMetadata`'s component list: the identifier of + /// the component's root object, the name it is known by, and the file it + /// lives in — which carries an identifier suffix often enough that the file + /// name is not a way to find it. + struct ComponentInfo final { + std::uint64_t identifier{}; + std::string name; + std::string locator; + }; + + const abstract::ReadableFilesystem *m_filesystem{nullptr}; + std::vector m_component_infos; + std::deque m_components; + std::unordered_map m_objects; + + const Component &load_(const ComponentInfo &info); +}; + +/// Reads @p path off @p filesystem and undoes its `.iwa` framing. +std::string read_iwa(const abstract::ReadableFilesystem &filesystem, + const AbsPath &path); + +/// Splits a decompressed `.iwa` into its objects, over @p data. +std::vector read_objects(std::string_view data); + +} // namespace odr::internal::iwork diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0161fc4b..96063143 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(odr_test "src/internal/svg/svg_file_test.cpp" "src/internal/xml/xml_file_test.cpp" + "src/internal/iwork/iwork_archive_test.cpp" "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" diff --git a/test/src/internal/iwork/iwork_archive_test.cpp b/test/src/internal/iwork/iwork_archive_test.cpp new file mode 100644 index 00000000..192e02cb --- /dev/null +++ b/test/src/internal/iwork/iwork_archive_test.cpp @@ -0,0 +1,120 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace odr::internal::iwork; + +namespace { + +std::string varint(std::uint64_t value) { + std::string result; + for (;;) { + const auto byte = static_cast(value & 0x7f); + value >>= 7; + result.push_back(value == 0 ? byte : static_cast(byte | 0x80)); + if (value == 0) { + return result; + } + } +} + +std::string number_field(const std::uint32_t number, + const std::uint64_t value) { + return varint(number << 3) + varint(value); +} + +std::string message_field(const std::uint32_t number, + const std::string &bytes) { + return varint((number << 3) | 2) + varint(bytes.size()) + bytes; +} + +/// `TSP.ArchiveInfo`: an identifier and one `MessageInfo` per payload message. +std::string archive_info( + const std::uint64_t identifier, + const std::vector> &messages) { + std::string result = number_field(1, identifier); + for (const auto &[type, length] : messages) { + result += message_field(2, number_field(1, type) + number_field(3, length)); + } + return result; +} + +std::string +object(const std::uint64_t identifier, + const std::vector> &messages, + const std::string &payload) { + const std::string info = archive_info(identifier, messages); + return varint(info.size()) + info + payload; +} + +} // namespace + +TEST(ReadObjects, one_object) { + const std::string data = object(1, {{10000, 5}}, "hello"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].identifier, 1); + EXPECT_EQ(objects[0].type, 10000); + EXPECT_EQ(objects[0].payload, "hello"); +} + +TEST(ReadObjects, objects_follow_one_another) { + const std::string data = + object(1, {{10000, 5}}, "hello") + object(1732514, {{2001, 5}}, "world"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[1].identifier, 1732514); + EXPECT_EQ(objects[1].type, 2001); + EXPECT_EQ(objects[1].payload, "world"); +} + +// An object may hold more than one message; only the first is modelled, and +// the length of the rest is what keeps the reader in step. +TEST(ReadObjects, later_messages_are_skipped) { + const std::string data = object(1732594, {{6247, 3}, {6247, 3}}, "onetwo") + + object(2, {{222, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 2); + EXPECT_EQ(objects[0].type, 6247); + EXPECT_EQ(objects[0].payload, "one"); + EXPECT_EQ(objects[1].identifier, 2); +} + +// There is no spec and no schema registry, so a type we have not mapped is an +// app version we have not seen — the reader keeps it and moves on. +TEST(ReadObjects, unknown_type_is_kept) { + const std::string data = object(7, {{123456, 1}}, "x"); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_EQ(objects[0].type, 123456); +} + +TEST(ReadObjects, empty_payload) { + const std::string data = object(1732550, {{3047, 0}}, ""); + + const std::vector objects = read_objects(data); + ASSERT_EQ(objects.size(), 1); + EXPECT_TRUE(objects[0].payload.empty()); +} + +TEST(ReadObjects, empty_component) { EXPECT_TRUE(read_objects("").empty()); } + +TEST(ReadObjects, archive_info_runs_past_the_component) { + const std::string data = object(1, {{10000, 5}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data.substr(0, 4))); +} + +TEST(ReadObjects, payload_runs_past_the_component) { + const std::string data = object(1, {{10000, 500}}, "hello"); + EXPECT_ANY_THROW(std::ignore = read_objects(data)); +} From e571f7f5b047aaaff8359fef728e9ce1897f4234 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 17:55:48 +0200 Subject: [PATCH 4/7] feat(iwork): name the three apple iwork file types `.pages`, `.numbers` and `.key` are zips, so today they are reported as `[zip]` and open as an archive rather than a document. Naming them gives a caller the extensions and MIME types to route one and hand a file picker, which is what has to be decided before the file is held. Classification only for now: which app wrote a package is read off its root archive, and only `.pages` has a fixture to pin that against, so the rows declare no capabilities and nothing detects or decodes one yet. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- apple/include/OdrCoreObjC/ODRFile.h | 4 ++ apple/src/ODRFile.mm | 4 ++ jni/java/app/opendocument/core/FileType.java | 3 +- python/src/bind_file.cpp | 5 ++- src/odr/file.hpp | 8 ++++ src/odr/internal/file_type_table.cpp | 42 ++++++++++++++++++++ test/src/odr_test.cpp | 2 +- 7 files changed, 65 insertions(+), 3 deletions(-) diff --git a/apple/include/OdrCoreObjC/ODRFile.h b/apple/include/OdrCoreObjC/ODRFile.h index d1b07078..ac19d087 100644 --- a/apple/include/OdrCoreObjC/ODRFile.h +++ b/apple/include/OdrCoreObjC/ODRFile.h @@ -83,6 +83,10 @@ typedef NS_ENUM(NSInteger, ODRFileType) { ODRFileTypeEnhancedMetafile, ODRFileTypeXml, + + ODRFileTypeIworkPages, + ODRFileTypeIworkNumbers, + ODRFileTypeIworkKeynote, } NS_SWIFT_NAME(FileType); typedef NS_ENUM(NSInteger, ODRFileCategory) { diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index 472f05f8..d616b1ab 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -92,6 +92,10 @@ ODR_SAME_ENUM(ODRFileTypeEnhancedMetafile, odr::FileType::enhanced_metafile); ODR_SAME_ENUM(ODRFileTypeXml, odr::FileType::xml); +ODR_SAME_ENUM(ODRFileTypeIworkPages, odr::FileType::iwork_pages); +ODR_SAME_ENUM(ODRFileTypeIworkNumbers, odr::FileType::iwork_numbers); +ODR_SAME_ENUM(ODRFileTypeIworkKeynote, odr::FileType::iwork_keynote); + ODR_SAME_ENUM(ODRFileCategoryUnknown, odr::FileCategory::unknown); ODR_SAME_ENUM(ODRFileCategoryText, odr::FileCategory::text); ODR_SAME_ENUM(ODRFileCategoryImage, odr::FileCategory::image); diff --git a/jni/java/app/opendocument/core/FileType.java b/jni/java/app/opendocument/core/FileType.java index 80d13963..74b1f5e1 100644 --- a/jni/java/app/opendocument/core/FileType.java +++ b/jni/java/app/opendocument/core/FileType.java @@ -16,7 +16,8 @@ public enum FileType { OGG_AUDIO, WAVEFORM_AUDIO, FREE_LOSSLESS_AUDIO_CODEC, MPEG4_VIDEO, QUICKTIME_VIDEO, THIRD_GENERATION_PARTNERSHIP_VIDEO, MATROSKA_VIDEO, AUDIO_VIDEO_INTERLEAVE, SCALABLE_VECTOR_GRAPHICS, WINDOWS_ICON, JPEG_XL, - JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML; + JPEG_2000, PHOTOSHOP_DOCUMENT, WINDOWS_METAFILE, ENHANCED_METAFILE, XML, + IWORK_PAGES, IWORK_NUMBERS, IWORK_KEYNOTE; static FileType fromNative(int code) { return code < 0 ? null : values()[code]; diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index ac48ef54..0d1163d8 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -83,7 +83,10 @@ void odr_python::bind_file(py::module_ &m) { .value("photoshop_document", odr::FileType::photoshop_document) .value("windows_metafile", odr::FileType::windows_metafile) .value("enhanced_metafile", odr::FileType::enhanced_metafile) - .value("xml", odr::FileType::xml); + .value("xml", odr::FileType::xml) + .value("iwork_pages", odr::FileType::iwork_pages) + .value("iwork_numbers", odr::FileType::iwork_numbers) + .value("iwork_keynote", odr::FileType::iwork_keynote); py::enum_(m, "FileCategory") .value("unknown", odr::FileCategory::unknown) diff --git a/src/odr/file.hpp b/src/odr/file.hpp index 45aea937..930be535 100644 --- a/src/odr/file.hpp +++ b/src/odr/file.hpp @@ -149,6 +149,14 @@ enum class FileType { // `[text_file, xml, scalable_vector_graphics]`. // https://en.wikipedia.org/wiki/XML xml, + + // https://en.wikipedia.org/wiki/IWork + iwork_pages, + // Classification only - `.numbers` and `.key` sit in the same package the + // pages engine reads, but which app wrote one is read off its root archive + // and no fixture pins those two, so nothing detects or decodes them yet. + iwork_numbers, + iwork_keynote, }; /// @brief Collection of file categories. diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 61c6ecdd..c5235642 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -250,6 +250,24 @@ constexpr std::array avi_mimetypes{"video/x-msvideo"sv, "video/avi"sv, // `decrypt` on an OOXML document type means a password-protected package, // detected as `office_open_xml_encrypted` and decrypting into the type named // here. ODF files decrypt in place and keep their type. +constexpr std::array pages_extensions{"pages"sv}; +constexpr std::array pages_mimetypes{ + "application/vnd.apple.pages"sv, + "application/x-iwork-pages-sffpages"sv, +}; + +constexpr std::array numbers_extensions{"numbers"sv}; +constexpr std::array numbers_mimetypes{ + "application/vnd.apple.numbers"sv, + "application/x-iwork-numbers-sffnumbers"sv, +}; + +constexpr std::array keynote_extensions{"key"sv}; +constexpr std::array keynote_mimetypes{ + "application/vnd.apple.keynote"sv, + "application/x-iwork-keynote-sffkey"sv, +}; + constexpr std::array table{ Row{FileType::unknown, "unknown"sv, @@ -755,6 +773,30 @@ constexpr std::array table{ .open = true, .translate_html = true, .color_scheme = true}}, + + // Classified so a caller can name the three and hand their MIME types to a + // file picker; no engine reads one yet. + Row{FileType::iwork_pages, + "pages"sv, + pages_extensions, + pages_mimetypes, + FileCategory::document, + DocumentType::text, + {}}, + Row{FileType::iwork_numbers, + "numbers"sv, + numbers_extensions, + numbers_mimetypes, + FileCategory::document, + DocumentType::spreadsheet, + {}}, + Row{FileType::iwork_keynote, + "key"sv, + keynote_extensions, + keynote_mimetypes, + FileCategory::document, + DocumentType::presentation, + {}}, }; /// Finds the row whose list, selected by @p list, contains @p needle. diff --git a/test/src/odr_test.cpp b/test/src/odr_test.cpp index 11bd2d6e..1f7a5fd2 100644 --- a/test/src/odr_test.cpp +++ b/test/src/odr_test.cpp @@ -27,7 +27,7 @@ namespace { std::vector every_file_type() { std::vector result; for (auto i = static_cast(FileType::unknown); - i <= static_cast(FileType::xml); ++i) { + i <= static_cast(FileType::iwork_keynote); ++i) { result.push_back(static_cast(i)); } return result; From 1ee90b6d5c64f8093e165fe559b076d98b0e75d9 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:02:06 +0200 Subject: [PATCH 5/7] feat(pages): open a `.pages` document and read its body text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.pages` package now opens as a text document rather than as the zip it is made of, and its body comes out as paragraphs. Which app wrote the package is read off the type of the root archive in `Index/Document.iwa` — the extension is not consulted, since a caller may not have one. Paragraph boundaries come from the storage's paragraph run table rather than from splitting the text on `\n`, and `U+2028` inside a paragraph becomes a line break. The anchor a drawable leaves in the text is dropped: styles, page geometry, drawables, images and tables are all still to come, so this is the text and nothing else. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- CMakeLists.txt | 4 + src/odr/exceptions.cpp | 2 + src/odr/exceptions.hpp | 5 + src/odr/internal/file_type_table.cpp | 9 +- src/odr/internal/iwork/iwork_document.cpp | 185 +++++++++++++++ src/odr/internal/iwork/iwork_document.hpp | 27 +++ .../internal/iwork/iwork_element_registry.cpp | 96 ++++++++ .../internal/iwork/iwork_element_registry.hpp | 52 +++++ src/odr/internal/iwork/iwork_file.cpp | 94 ++++++++ src/odr/internal/iwork/iwork_file.hpp | 41 ++++ src/odr/internal/iwork/iwork_parser.cpp | 220 ++++++++++++++++++ src/odr/internal/iwork/iwork_parser.hpp | 18 ++ src/odr/internal/iwork/iwork_types.hpp | 49 ++++ src/odr/internal/open_strategy.cpp | 34 +++ test/CMakeLists.txt | 1 + test/src/internal/iwork/pages_test.cpp | 127 ++++++++++ 16 files changed, 961 insertions(+), 3 deletions(-) create mode 100644 src/odr/internal/iwork/iwork_document.cpp create mode 100644 src/odr/internal/iwork/iwork_document.hpp create mode 100644 src/odr/internal/iwork/iwork_element_registry.cpp create mode 100644 src/odr/internal/iwork/iwork_element_registry.hpp create mode 100644 src/odr/internal/iwork/iwork_file.cpp create mode 100644 src/odr/internal/iwork/iwork_file.hpp create mode 100644 src/odr/internal/iwork/iwork_parser.cpp create mode 100644 src/odr/internal/iwork/iwork_parser.hpp create mode 100644 src/odr/internal/iwork/iwork_types.hpp create mode 100644 test/src/internal/iwork/pages_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fed40439..ab090b17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,10 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/xml_file.cpp" "src/odr/internal/iwork/iwork_archive.cpp" + "src/odr/internal/iwork/iwork_document.cpp" + "src/odr/internal/iwork/iwork_element_registry.cpp" + "src/odr/internal/iwork/iwork_file.cpp" + "src/odr/internal/iwork/iwork_parser.cpp" "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" diff --git a/src/odr/exceptions.cpp b/src/odr/exceptions.cpp index aaa8d361..c94619a8 100644 --- a/src/odr/exceptions.cpp +++ b/src/odr/exceptions.cpp @@ -68,6 +68,8 @@ NoFontFile::NoFontFile() : Exception("not a font file") {} NoLegacyMicrosoftFile::NoLegacyMicrosoftFile() : Exception("not a legacy microsoft office file") {} +NoIworkFile::NoIworkFile() : Exception("not an iwork file") {} + NoXmlFile::NoXmlFile() : Exception("not an xml file") {} NoSvgFile::NoSvgFile() : Exception("not an svg file") {} diff --git a/src/odr/exceptions.hpp b/src/odr/exceptions.hpp index c9d55fc1..597bc863 100644 --- a/src/odr/exceptions.hpp +++ b/src/odr/exceptions.hpp @@ -142,6 +142,11 @@ struct NoLegacyMicrosoftFile final : Exception { NoLegacyMicrosoftFile(); }; +/// @brief No iWork file exception +struct NoIworkFile final : Exception { + NoIworkFile(); +}; + /// @brief No XML file exception struct NoXmlFile final : Exception { NoXmlFile(); diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index c5235642..5333423e 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -774,15 +774,18 @@ constexpr std::array table{ .translate_html = true, .color_scheme = true}}, - // Classified so a caller can name the three and hand their MIME types to a - // file picker; no engine reads one yet. Row{FileType::iwork_pages, "pages"sv, pages_extensions, pages_mimetypes, FileCategory::document, DocumentType::text, - {}}, + {.detect_by_content = true, + .open = true, + .translate_html = true, + .color_scheme = true}}, + // Classified so a caller can name these two and hand their MIME types to a + // file picker; no engine reads either yet. Row{FileType::iwork_numbers, "numbers"sv, numbers_extensions, diff --git a/src/odr/internal/iwork/iwork_document.cpp b/src/odr/internal/iwork/iwork_document.cpp new file mode 100644 index 00000000..5d6848f7 --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.cpp @@ -0,0 +1,185 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace odr::internal::iwork { + +namespace { +std::unique_ptr +create_element_adapter(ElementRegistry ®istry); +} + +Document::Document(std::shared_ptr files) + : internal::Document(FileType::iwork_pages, DocumentType::text, + std::move(files)) { + m_root_element = parse_pages_tree(m_element_registry, *m_files); + + m_element_adapter = create_element_adapter(m_element_registry); +} + +const ElementRegistry &Document::element_registry() const { + return m_element_registry; +} + +bool Document::is_editable() const noexcept { return false; } + +bool Document::is_savable(const bool encrypted) const noexcept { + (void)encrypted; + return false; +} + +void Document::save(const Path &path) const { + (void)path; + throw UnsupportedOperation(); +} + +void Document::save(const Path &path, const char *password) const { + (void)path; + (void)password; + throw UnsupportedOperation(); +} + +namespace { + +class ElementAdapter final : public abstract::ElementAdapter, + public abstract::TextRootAdapter, + public abstract::LineBreakAdapter, + public abstract::ParagraphAdapter, + public abstract::TextAdapter { +public: + explicit ElementAdapter(ElementRegistry ®istry) : m_registry(®istry) {} + + [[nodiscard]] ElementType + element_type(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).type; + } + + [[nodiscard]] ElementIdentifier + element_parent(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).parent_id; + } + [[nodiscard]] ElementIdentifier + element_first_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).first_child_id; + } + [[nodiscard]] ElementIdentifier + element_last_child(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).last_child_id; + } + [[nodiscard]] ElementIdentifier + element_previous_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).previous_sibling_id; + } + [[nodiscard]] ElementIdentifier + element_next_sibling(const ElementIdentifier element_id) const override { + return m_registry->element_at(element_id).next_sibling_id; + } + + [[nodiscard]] bool + element_is_unique(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_self_locatable(const ElementIdentifier element_id) const override { + (void)element_id; + return true; + } + [[nodiscard]] bool + element_is_editable(const ElementIdentifier element_id) const override { + (void)element_id; + return false; + } + [[nodiscard]] DocumentPath + element_document_path(const ElementIdentifier element_id) const override { + return util::document::extract_path(*this, element_id, null_element_id); + } + [[nodiscard]] ElementIdentifier + element_navigate_path(const ElementIdentifier element_id, + const DocumentPath &path) const override { + return util::document::navigate_path(*this, element_id, path); + } + + [[nodiscard]] const TextRootAdapter * + text_root_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::root ? this : nullptr; + } + [[nodiscard]] const LineBreakAdapter * + line_break_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::line_break ? this : nullptr; + } + [[nodiscard]] const ParagraphAdapter * + paragraph_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::paragraph ? this : nullptr; + } + [[nodiscard]] const TextAdapter * + text_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::text ? this : nullptr; + } + + // The page geometry sits in the document archive and the styles in + // `Index/DocumentStylesheet.iwa`; neither is read yet. + [[nodiscard]] PageLayout + text_root_page_layout(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] ElementIdentifier text_root_first_master_page( + const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] TextStyle + line_break_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] ParagraphStyle + paragraph_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + [[nodiscard]] TextStyle + paragraph_text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + + [[nodiscard]] std::string + text_content(const ElementIdentifier element_id) const override { + return m_registry->text_element_at(element_id).text; + } + void text_set_content(const ElementIdentifier element_id, + const std::string &text) const override { + (void)element_id; + (void)text; + throw UnsupportedOperation(); + } + [[nodiscard]] TextStyle + text_style(const ElementIdentifier element_id) const override { + (void)element_id; + return {}; + } + +private: + ElementRegistry *m_registry{nullptr}; +}; + +std::unique_ptr +create_element_adapter(ElementRegistry ®istry) { + return std::make_unique(registry); +} + +} // namespace + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_document.hpp b/src/odr/internal/iwork/iwork_document.hpp new file mode 100644 index 00000000..d73c8b0d --- /dev/null +++ b/src/odr/internal/iwork/iwork_document.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include + +namespace odr::internal::iwork { + +/// A `.pages` package, read as a text document. +class Document final : public internal::Document { +public: + explicit Document(std::shared_ptr files); + + [[nodiscard]] const ElementRegistry &element_registry() const; + + [[nodiscard]] bool is_editable() const noexcept override; + [[nodiscard]] bool is_savable(bool encrypted) const noexcept override; + + void save(const Path &path) const override; + void save(const Path &path, const char *password) const override; + +private: + ElementRegistry m_element_registry; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.cpp b/src/odr/internal/iwork/iwork_element_registry.cpp new file mode 100644 index 00000000..2ce6952f --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.cpp @@ -0,0 +1,96 @@ +#include + +#include + +namespace odr::internal::iwork { + +void ElementRegistry::clear() noexcept { + m_elements.clear(); + m_texts.clear(); +} + +[[nodiscard]] std::size_t ElementRegistry::size() const noexcept { + return m_elements.size(); +} + +std::tuple +ElementRegistry::create_element(const ElementType type) { + Element &element = m_elements.emplace_back(); + ElementIdentifier element_id = m_elements.size(); + element.type = type; + return {element_id, element}; +} + +std::tuple +ElementRegistry::create_text_element() { + const auto &[element_id, element] = create_element(ElementType::text); + auto [it, success] = m_texts.emplace(element_id, Text{}); + return {element_id, element, it->second}; +} + +ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) { + check_element_id(id); + return m_elements.at(id - 1); +} + +ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) { + check_text_id(id); + return m_texts.at(id); +} + +const ElementRegistry::Element & +ElementRegistry::element_at(const ElementIdentifier id) const { + check_element_id(id); + return m_elements.at(id - 1); +} + +const ElementRegistry::Text & +ElementRegistry::text_element_at(const ElementIdentifier id) const { + check_text_id(id); + return m_texts.at(id); +} + +void ElementRegistry::append_child(const ElementIdentifier parent_id, + const ElementIdentifier child_id) { + check_element_id(parent_id); + check_element_id(child_id); + if (element_at(child_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_child: child already has a parent"); + } + + const ElementIdentifier previous_sibling_id = + element_at(parent_id).last_child_id; + + element_at(child_id).parent_id = parent_id; + element_at(child_id).previous_sibling_id = previous_sibling_id; + + if (element_at(parent_id).first_child_id == null_element_id) { + element_at(parent_id).first_child_id = child_id; + } else { + element_at(previous_sibling_id).next_sibling_id = child_id; + } + element_at(parent_id).last_child_id = child_id; +} + +void ElementRegistry::check_element_id(const ElementIdentifier id) const { + if (id == null_element_id) { + throw std::out_of_range("ElementRegistry::check_id: null identifier"); + } + if (id - 1 >= m_elements.size()) { + throw std::out_of_range( + "ElementRegistry::check_id: identifier out of range"); + } +} + +void ElementRegistry::check_text_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_texts.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.hpp b/src/odr/internal/iwork/iwork_element_registry.hpp new file mode 100644 index 00000000..a7a7bbaa --- /dev/null +++ b/src/odr/internal/iwork/iwork_element_registry.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace odr::internal::iwork { + +class ElementRegistry final { +public: + struct Element final { + ElementIdentifier parent_id{null_element_id}; + ElementIdentifier first_child_id{null_element_id}; + ElementIdentifier last_child_id{null_element_id}; + ElementIdentifier previous_sibling_id{null_element_id}; + ElementIdentifier next_sibling_id{null_element_id}; + ElementType type{ElementType::none}; + }; + + struct Text final { + std::string text; + }; + + void clear() noexcept; + + [[nodiscard]] std::size_t size() const noexcept; + + std::tuple create_element(ElementType type); + std::tuple create_text_element(); + + [[nodiscard]] Element &element_at(ElementIdentifier id); + [[nodiscard]] Text &text_element_at(ElementIdentifier id); + + [[nodiscard]] const Element &element_at(ElementIdentifier id) const; + [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; + + void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + +private: + std::vector m_elements; + std::unordered_map m_texts; + + void check_element_id(ElementIdentifier id) const; + void check_text_id(ElementIdentifier id) const; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_file.cpp b/src/odr/internal/iwork/iwork_file.cpp new file mode 100644 index 00000000..22bcdfab --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// The type of the root archive says which app wrote the package. Only +/// `.pages` is pinned — a `.numbers` or `.key` fixture would be needed to read +/// theirs off, and the extension is not an answer. +FileType file_type_by_archive_type(const std::uint32_t type) { + switch (type) { + case iwork::archive_type::pages_document: + return FileType::iwork_pages; + default: + return FileType::unknown; + } +} + +/// Reads the root archive of the package's `Document` component. The component +/// list in `Index/Metadata.iwa` is not consulted: this runs on every zip a +/// caller opens, and the `Document` component is the one whose file name never +/// carries an identifier suffix. +FileType parse_file_type(const abstract::ReadableFilesystem &filesystem) { + const std::string data = + iwork::read_iwa(filesystem, AbsPath("/Index/Document.iwa")); + const std::vector objects = iwork::read_objects(data); + if (objects.empty()) { + throw NoIworkFile(); + } + + const FileType file_type = file_type_by_archive_type(objects.front().type); + if (file_type == FileType::unknown) { + throw NoIworkFile(); + } + return file_type; +} + +} // namespace + +iwork::IworkFile::IworkFile( + std::shared_ptr filesystem) + : m_filesystem{std::move(filesystem)} { + if (!m_filesystem->is_file(AbsPath("/Index/Document.iwa"))) { + throw NoIworkFile(); + } + + m_file_meta.type = parse_file_type(*m_filesystem); + m_file_meta.mimetype = mimetype_by_file_type(m_file_meta.type); + m_file_meta.document_type = document_type_by_file_type(m_file_meta.type); +} + +std::shared_ptr iwork::IworkFile::file() const noexcept { + return {}; +} + +FileType iwork::IworkFile::file_type() const noexcept { + return m_file_meta.type; +} + +std::string_view iwork::IworkFile::mimetype() const noexcept { + return m_file_meta.mimetype; +} + +FileMeta iwork::IworkFile::file_meta() const noexcept { return m_file_meta; } + +DocumentType iwork::IworkFile::document_type() const { + return m_file_meta.document_type; +} + +bool iwork::IworkFile::is_decodable() const noexcept { return true; } + +std::shared_ptr iwork::IworkFile::document() const { + switch (file_type()) { + case FileType::iwork_pages: + return std::make_shared(m_filesystem); + default: + throw UnsupportedFileType(file_type()); + } +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_file.hpp b/src/odr/internal/iwork/iwork_file.hpp new file mode 100644 index 00000000..ce86d519 --- /dev/null +++ b/src/odr/internal/iwork/iwork_file.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace odr::internal::abstract { +class Document; +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { + +/// An iWork package (`.pages`, `.numbers`, `.key`). Which app wrote it is read +/// off the root archive of `Index/Document.iwa`, not off the file name, which +/// a caller may have lost. +class IworkFile final : public abstract::DocumentFile { +public: + explicit IworkFile(std::shared_ptr filesystem); + + [[nodiscard]] std::shared_ptr file() const noexcept override; + + [[nodiscard]] FileType file_type() const noexcept override; + [[nodiscard]] std::string_view mimetype() const noexcept override; + [[nodiscard]] FileMeta file_meta() const noexcept override; + + [[nodiscard]] DocumentType document_type() const override; + + [[nodiscard]] bool is_decodable() const noexcept override; + + [[nodiscard]] std::shared_ptr document() const override; + +private: + std::shared_ptr m_filesystem; + FileMeta m_file_meta; +}; + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp new file mode 100644 index 00000000..20056440 --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -0,0 +1,220 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace odr::internal { + +namespace { + +/// `U+2028 LINE SEPARATOR` — a line break inside a paragraph. +constexpr std::string_view line_separator = "\xe2\x80\xa8"; +/// `U+FFFC OBJECT REPLACEMENT CHARACTER` — where a drawable is anchored in the +/// text. Nothing reads drawables yet, so the anchor is dropped rather than +/// rendered as a glyph. +constexpr std::string_view object_replacement = "\xef\xbf\xbc"; + +/// The byte length of the UTF-8 sequence @p lead starts. +std::size_t utf8_length(const std::uint8_t lead) { + if (lead < 0x80) { + return 1; + } + if ((lead & 0xe0) == 0xc0) { + return 2; + } + if ((lead & 0xf0) == 0xe0) { + return 3; + } + if ((lead & 0xf8) == 0xf0) { + return 4; + } + throw std::runtime_error("iwork: text is not utf-8"); +} + +/// Translates the ascending UTF-16 code unit @p indices a storage's run tables +/// count in into byte offsets into @p text. +std::vector +utf16_offsets(const std::string_view text, + const std::vector &indices) { + std::vector result; + result.reserve(indices.size()); + + std::size_t offset = 0; + std::uint64_t unit = 0; + auto next = indices.begin(); + + for (;;) { + while (next != indices.end() && *next == unit) { + result.push_back(offset); + ++next; + } + if (next == indices.end()) { + return result; + } + if (offset >= text.size()) { + throw std::runtime_error("iwork: run table points past the text"); + } + + const std::size_t length = + utf8_length(static_cast(text[offset])); + if (offset + length > text.size()) { + throw std::runtime_error("iwork: text ends mid-character"); + } + offset += length; + // everything outside the basic multilingual plane is a surrogate pair + unit += length == 4 ? 2 : 1; + } +} + +/// The character index each paragraph of @p storage starts at. Paragraph +/// boundaries are the run table's rather than every `\n` in the text — the two +/// agree today, but the table is what says so. +std::vector paragraph_starts(const iwork::Message &storage) { + std::vector result; + + const std::optional table = + storage.bytes_field(iwork::text_storage::paragraph_styles); + if (!table.has_value()) { + return {0}; + } + + for (const iwork::Field &entry : + iwork::Message(*table).repeated_field(iwork::attribute_table::entries)) { + if (entry.type != iwork::WireType::length_delimited) { + throw std::runtime_error("iwork: malformed paragraph style table"); + } + const iwork::Message run(entry.bytes); + result.push_back( + run.number_field(iwork::attribute_table_entry::character_index) + .value_or(0)); + } + + // an empty document carries an empty table; either way the body starts at + // its first character + if (result.empty() || result.front() != 0) { + result.insert(result.begin(), 0); + } + return result; +} + +/// @p text without the drawable anchors it holds. +std::string without_anchors(const std::string_view text) { + std::string result; + result.reserve(text.size()); + + for (std::size_t position = 0; position < text.size();) { + const std::size_t anchor = text.find(object_replacement, position); + if (anchor == std::string_view::npos) { + result += text.substr(position); + break; + } + result += text.substr(position, anchor - position); + position = anchor + object_replacement.size(); + } + return result; +} + +/// Fills @p paragraph_id with the text of one paragraph, breaking it at the +/// line separators it holds. +void parse_paragraph(iwork::ElementRegistry ®istry, + const ElementIdentifier paragraph_id, + std::string_view content) { + const auto append_text = [&](const std::string_view part) { + std::string text = without_anchors(part); + if (text.empty()) { + return; + } + auto [text_id, element, payload] = registry.create_text_element(); + payload.text = std::move(text); + registry.append_child(paragraph_id, text_id); + }; + + for (std::size_t position = content.find(line_separator); + position != std::string_view::npos; + position = content.find(line_separator)) { + append_text(content.substr(0, position)); + + auto [break_id, element] = registry.create_element(ElementType::line_break); + registry.append_child(paragraph_id, break_id); + + content.remove_prefix(position + line_separator.size()); + } + append_text(content); +} + +} // namespace + +ElementIdentifier +iwork::parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files) { + Package package(files); + + const std::vector &objects = package.component("Document").objects(); + if (objects.empty() || objects.front().type != archive_type::pages_document) { + throw std::runtime_error("iwork: no pages document archive"); + } + + const Message document(objects.front().payload); + const std::optional body = + document.bytes_field(document_archive::body_storage); + if (!body.has_value()) { + throw std::runtime_error("iwork: document archive holds no body"); + } + const std::optional body_identifier = + Message(*body).number_field(reference::identifier); + if (!body_identifier.has_value()) { + throw std::runtime_error("iwork: body reference names no object"); + } + + const Object &body_object = package.object(*body_identifier); + if (body_object.type != archive_type::text_storage) { + throw std::runtime_error("iwork: body is not a text storage"); + } + const Message storage(body_object.payload); + + // the text arrives as a small number of large strings; the run tables index + // it as one + std::string text; + for (const Field &part : storage.repeated_field(text_storage::text)) { + if (part.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed text storage"); + } + text += part.bytes; + } + + const std::vector starts = + utf16_offsets(text, paragraph_starts(storage)); + + auto [root_id, root] = registry.create_element(ElementType::root); + + const std::string_view body_text(text); + for (std::size_t i = 0; i < starts.size(); ++i) { + const std::size_t begin = starts[i]; + const std::size_t end = i + 1 < starts.size() ? starts[i + 1] : text.size(); + + std::string_view content = body_text.substr(begin, end - begin); + // the paragraph mark belongs to the paragraph it ends, and the last + // paragraph of a body does not carry one + if (content.ends_with('\n')) { + content.remove_suffix(1); + } + + auto [paragraph_id, paragraph] = + registry.create_element(ElementType::paragraph); + registry.append_child(root_id, paragraph_id); + parse_paragraph(registry, paragraph_id, content); + } + + return root_id; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_parser.hpp b/src/odr/internal/iwork/iwork_parser.hpp new file mode 100644 index 00000000..82cbd72c --- /dev/null +++ b/src/odr/internal/iwork/iwork_parser.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace odr::internal::abstract { +class ReadableFilesystem; +} // namespace odr::internal::abstract + +namespace odr::internal::iwork { +class ElementRegistry; + +/// Parses the body of a `.pages` package into root → paragraph → text +/// elements. +/// \return the root element id. +ElementIdentifier parse_pages_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_types.hpp b/src/odr/internal/iwork/iwork_types.hpp new file mode 100644 index 00000000..7cf0f962 --- /dev/null +++ b/src/odr/internal/iwork/iwork_types.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace odr::internal::iwork { + +/// The archive types and field numbers the engine reads. +/// +/// There is no spec and Apple has never published the `.proto` schemas, so +/// each of these is cited to the fixture it was read off rather than to a +/// section number, and holds for the iWork version that wrote it — see +/// `Metadata/BuildVersionHistory.plist`. A type id that is not here is one we +/// have not mapped, which the reader skips rather than throws on. +namespace archive_type { +/// `TP.DocumentArchive`, the root of a `.pages` package. +/// `empty.pages Index/Document.iwa` object 1 (iWork 13.2). +constexpr std::uint32_t pages_document = 10000; +/// `TSWP.StorageArchive`, a run of text with its run tables. +/// `empty.pages Index/Document.iwa` object 1732514 (iWork 13.2). +constexpr std::uint32_t text_storage = 2001; +} // namespace archive_type + +namespace document_archive { +/// The body text storage, as a `TSP.Reference`. +constexpr std::uint32_t body_storage = 4; +} // namespace document_archive + +namespace text_storage { +/// The text, in a small number of large strings. +constexpr std::uint32_t text = 3; +/// The paragraph style run table: one entry per paragraph, holding the +/// character index the paragraph starts at and, where it has one, its style. +constexpr std::uint32_t paragraph_styles = 5; +} // namespace text_storage + +/// A run table parallel to the text, as `TSWP.ObjectAttributeTable`. +namespace attribute_table { +constexpr std::uint32_t entries = 1; +} // namespace attribute_table + +namespace reference { +constexpr std::uint32_t identifier = 1; +} // namespace reference + +namespace attribute_table_entry { +constexpr std::uint32_t character_index = 1; +} // namespace attribute_table_entry + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 68a4e260..4a53a4ad 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,18 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoOpenDocumentFile(); } + if (as == FileType::iwork_pages) { + ODR_VERBOSE(logger, "open as iwork"); + try { + auto zip_file = std::make_unique(file); + auto filesystem = zip_file->archive()->as_filesystem(); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + throw NoIworkFile(); + } + if (as == FileType::office_open_xml_document || as == FileType::office_open_xml_presentation || as == FileType::office_open_xml_workbook || @@ -258,6 +271,13 @@ open_strategy::list_file_types(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + result.push_back(iwork::IworkFile(filesystem).file_type()); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } catch (...) { ODR_VERBOSE(logger, "failed to open as zip"); } @@ -367,6 +387,13 @@ open_strategy::open_file(const std::shared_ptr &file, ODR_VERBOSE(logger, "failed to open as ooxml"); } + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } + return zip_file; } if (file_type == FileType::compound_file_binary_format) { @@ -545,6 +572,13 @@ open_strategy::open_document_file(const std::shared_ptr &file, } catch (...) { ODR_VERBOSE(logger, "failed to open as ooxml"); } + + try { + ODR_VERBOSE(logger, "try open as iwork"); + return std::make_unique(filesystem); + } catch (...) { + ODR_VERBOSE(logger, "failed to open as iwork"); + } } else if (file_type == FileType::compound_file_binary_format) { ODR_VERBOSE(logger, "open as cbf"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 96063143..315e50d3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -62,6 +62,7 @@ add_executable(odr_test "src/internal/iwork/iwork_archive_test.cpp" "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" + "src/internal/iwork/pages_test.cpp" "src/internal/odf/odf_table_test.cpp" diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp new file mode 100644 index 00000000..820dfc95 --- /dev/null +++ b/test/src/internal/iwork/pages_test.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +using namespace odr; +using odr::test::TestData; + +namespace { + +/// The paragraphs of a text root, a line break reading as a newline. +std::vector paragraphs(const Element root) { + std::vector result; + + for (const Element paragraph : root.children()) { + EXPECT_EQ(paragraph.type(), ElementType::paragraph); + + std::string text; + for (const Element child : paragraph.children()) { + if (child.type() == ElementType::line_break) { + text += '\n'; + } else { + text += child.as_text().content(); + } + } + result.push_back(std::move(text)); + } + + return result; +} + +} // namespace + +TEST(Iwork, pages_is_detected_by_content) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + const std::string path = + TestData::test_file_path("odr-public/pages/style-various-1.pages"); + + EXPECT_THAT(list_file_types(path, logger), + testing::Contains(FileType::iwork_pages)); + + const DecodedFile file(path, logger); + EXPECT_EQ(file.file_type(), FileType::iwork_pages); + EXPECT_EQ(file.file_category(), FileCategory::document); + EXPECT_EQ(file.as_document_file().document_type(), DocumentType::text); +} + +// A document with nothing in it must come back with an empty body rather than +// throw: `empty.pages` carries a body storage that holds no text at all. +TEST(Iwork, pages_empty) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/empty.pages"), logger); + EXPECT_EQ(document_file.file_type(), FileType::iwork_pages); + + const Document document = document_file.document(); + EXPECT_EQ(document.document_type(), DocumentType::text); + EXPECT_FALSE(document.is_editable()); + EXPECT_FALSE(document.is_savable(false)); + + EXPECT_EQ(paragraphs(document.root_element()), + (std::vector{""})); +} + +TEST(Iwork, pages_body_text) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/style-various-1.pages"), + logger); + + const Document document = document_file.document(); + const std::vector text = paragraphs(document.root_element()); + + // one element per paragraph of the body, including the empty ones that + // separate its sections + ASSERT_EQ(text.size(), 54); + EXPECT_EQ(text[0], "Table of Contents"); + // the anchor of a drawable is dropped: nothing reads drawables yet + EXPECT_EQ(text[1], ""); + EXPECT_EQ(text[4], "Headline"); + EXPECT_EQ(text[5], "Nested Headline"); + EXPECT_EQ(text[7], "Text"); + EXPECT_EQ(text[9], "Hyperlink google"); + EXPECT_EQ(text[11], "Default"); + EXPECT_EQ(text[12], "Bold"); + EXPECT_EQ(text.back(), "image"); +} + +// Components share names — `style-various-1.pages` holds two dozen called +// `Tables/DataList` — so the package has to load them by locator. Keying on +// the name hands back the wrong file and leaves the rest never loaded, which +// shows up as an object nothing can resolve. +TEST(Iwork, package_resolves_across_components) { + using odr::internal::iwork::Package; + + const auto file = + std::make_shared(odr::internal::AbsPath( + TestData::test_file_path("odr-public/pages/style-various-1.pages"))); + const auto filesystem = + odr::internal::zip::ZipFile(file).archive()->as_filesystem(); + + Package package(*filesystem); + + EXPECT_EQ(package.component("Document").objects().front().identifier, 1); + // the stylesheet, which is a component of its own + EXPECT_EQ(package.object(1732588).identifier, 1732588); + // the root of a `Tables/DataList` that is not the first one of that name + EXPECT_EQ(package.object(1732940).identifier, 1732940); +} From 32284fd61c8db0b5ca68d51717b9eaa93ba86899 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:03:59 +0200 Subject: [PATCH 6/7] docs(iwork): record what the first two stages decided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PLAN.md` was written before any of it existed; this is the module's `AGENTS.md` alongside it — why a fixture is the citation here rather than a spec section, why snappy and protobuf are in-tree, the `Message` lifetime the whole engine rests on, and where the run tables sit. `PLAN.md` marks the two landed stages and the three places the plan and the code disagreed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- AGENTS.md | 10 +++ CHANGELOG.md | 6 ++ src/odr/internal/iwork/AGENTS.md | 109 +++++++++++++++++++++++++++++++ src/odr/internal/iwork/PLAN.md | 65 +++++++++++------- 4 files changed, 167 insertions(+), 23 deletions(-) create mode 100644 src/odr/internal/iwork/AGENTS.md diff --git a/AGENTS.md b/AGENTS.md index d7b27100..0eafe31f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/odf/` | OpenDocument (odt/ods/odp/odg); see [`odf/AGENTS.md`](src/odr/internal/odf/AGENTS.md). | | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | +| `src/odr/internal/iwork/` | Apple iWork (`.pages` today); see [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) + [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md). | | `src/odr/internal/pdf/` | PDF (own parser). | | `src/odr/internal/xml/` | XML, rendered as a source view; see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | @@ -218,6 +219,15 @@ Dispatch `release.yml` against main, publish the draft that appears — 4. Register the factory (e.g. `oldms_file.cpp::document()` switches on `file_type()`), add sources to `CMakeLists.txt`, add a GoogleTest. +## Apple iWork (`iwork`) + +`.pages` opens as a text document and renders its body text; `.numbers` and +`.key` are named but not decoded. There is no spec — the module cites fixtures +instead, keeps its own Snappy and protobuf readers, and fails soft on archive +types it has not mapped. Read [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) +before touching it, and [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md) for +what comes next. + ## Legacy Microsoft binary formats (`oldms`) CFB container handling exists; each format is a small module under `oldms/` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc85b98..678452ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- Apple iWork: a `.pages` file opens as a text document and renders its body + text, instead of coming back as the zip it is made of. Styles, page geometry, + images and tables are not read yet. `.numbers` and `.key` are named — + `FileType::iwork_numbers`, `FileType::iwork_keynote`, their extensions and + MIME types — but there is no decoder behind either. + ## v6.10.1 - 2026-08-21 - A linked image in a docx or xlsx (`embed_images = false`) is named relative diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md new file mode 100644 index 00000000..4223d55f --- /dev/null +++ b/src/odr/internal/iwork/AGENTS.md @@ -0,0 +1,109 @@ +# AGENTS.md — `internal/iwork` + +Read the root [`AGENTS.md`](../../../../AGENTS.md) first, then +[`PLAN.md`](PLAN.md), which is where this module is going and in what order. +This file is what the landed stages decided, and why. + +Landed: **stage 1** (detection and the container) and **stage 2** (Pages body +text). A `.pages` opens as a text document and renders its paragraphs. +Everything else in `PLAN.md` is still ahead. + +## There is no spec, so a fixture is the citation + +Apple has never published the `.proto` schemas and nothing is vendored under +`offline/documentation/`. Where `oldms/` writes `[MS-XLS] §2.4.1`, this module +writes `empty.pages Index/Document.iwa +0` — the byte layout verified against a +file in the repo is the only claim treated as fact. + +Everything the engine reads by number lives in `iwork_types.hpp`, each constant +cited to the fixture it was read off. Read `numbers-parser`, `keynote-parser`, +`obriensp/iWorkFileFormat` and `libetonyek` for facts; **copy code from none of +them**. + +**Fail soft on a type id we have not mapped, fail fast on broken framing.** The +root `AGENTS.md` says to throw where the spec dictates what to expect. Here +there is no spec, and an unknown type id means Apple shipped a version we have +not seen — a reader that throws on one cannot open next year's files. What does +throw: framing that overruns the file, a Snappy block that does not fill its +declared length, a varint that does not terminate, an identifier the package +does not hold, and text that is not UTF-8. + +## No new dependencies + +Two pieces would normally be a conan line each, and both would be wrong. + +- **Snappy** — the `.iwa` framing is Apple's own (`0x00`, a little-endian + 24-bit compressed length, repeated to EOF), not Snappy's stream framing, so + only the *block* decoder applies. `iwork_snappy.cpp` is that, in about a + hundred lines. +- **Protobuf** — only the wire format is needed, and with no schemas a code + generator has nothing to generate. Linking conan `protobuf` would drag it + into the wasm, android and apple builds to replace `iwork_protobuf.cpp`. + +Both stay inside `iwork/` until something else wants them; a wire reader with +one user has not earned a package. + +## `Message` views the buffer it was read from + +`iwork::Message` parses one level eagerly and leaves nested messages, strings +and packed fields as `std::string_view`s into the bytes it was handed. So the +buffer has to outlive it — `Component` owns its decompressed data behind a +`unique_ptr` for exactly that reason, and a `Message(some_temporary())` is a +dangling read rather than a compile error. + +## An `.iwa` is an object graph, not a tree + +A component file is a flat sequence of `(varint length, TSP.ArchiveInfo, +payload)`, and objects reference each other by identifier — across components. +So `Package` reads the component list from `Index/Metadata.iwa` first and +decompresses a component when something in it is asked for; `object(id)` loads +further components until the identifier turns up. Walking files in directory +order and hoping a tree falls out is the mistake to avoid. + +**Component names are not file names.** `Index/Metadata.iwa` maps a component's +name to its locator, and the locator carries an identifier suffix often enough +that globbing for `CalculationEngine.iwa` finds it in one fixture and not in +the other. + +The one place that skips the component list is detection: `IworkFile` reads +`/Index/Document.iwa` directly, because it runs on every zip a caller opens and +`Document` is the component whose file name never carries a suffix. + +## Which app wrote the package comes off the root archive + +`TP.DocumentArchive` is type 10000, verified on both `.pages` fixtures. The +extension is not consulted — a caller may have lost it — and neither is +`Metadata/Properties.plist`, which names an app version but not the app. + +That is also why only `.pages` is detected. `iwork_numbers` and `iwork_keynote` +have `file_type_table.cpp` rows so a caller can name them and hand a file +picker their MIME types, but no capabilities: reading their root archive types +off a guess is exactly what this module does not do, and neither has a fixture +in the test data yet. + +## Paragraphs come from the run table + +A `TSWP.StorageArchive` holds its text as a few large strings plus run tables +parallel to it — index/value pairs for paragraph styles, character styles and +attachments. Paragraph boundaries are the **paragraph style table's**, not +every `\n` in the text. The two agree on both fixtures, but the table is what +says so, and `U+2028` is a line break *inside* a paragraph rather than a +paragraph boundary. + +Run tables count in **UTF-16 code units** while the text is UTF-8, so +`iwork_parser.cpp` translates the indices in one pass over the text. An index +that lands mid-character is an error, not a rounding. + +`U+FFFC` is where a drawable is anchored. Nothing reads drawables yet, so the +anchor is dropped rather than rendered as a glyph — see stage 4. + +`empty.pages` is the regression that matters at this level: a body storage that +carries no text at all must produce an empty body, not an exception. + +## Not read yet + +`Index/DocumentStylesheet.iwa` (so `text_root_page_layout` is empty and every +style is the default), drawables and images, `Index/Tables/`, and everything +`PLAN.md` lists as deferred. `password_encrypted()` is not answered either: an +encrypted package is one whose `Index/Document.iwa` does not decompress, which +falls back to reporting the file as a zip. diff --git a/src/odr/internal/iwork/PLAN.md b/src/odr/internal/iwork/PLAN.md index f57831fc..a8bbf830 100644 --- a/src/odr/internal/iwork/PLAN.md +++ b/src/odr/internal/iwork/PLAN.md @@ -1,28 +1,24 @@ # iWork plan -Where an iwork module would go, and in what order. Written before stage 1; keep -it honest as stages land. +Where an iwork module goes, and in what order. Written before stage 1; kept +honest as stages land. **Stages 1 and 2 have landed** — see +[`AGENTS.md`](AGENTS.md) for what they decided. Stage 3 is next. ## Today -Nothing decodes, and unlike rtf there is not even a `FileType` yet. A `.pages` -is a zip, so `magic.cpp:95` reports `FileType::zip`, `list_file_types` probes -odf then ooxml (`open_strategy.cpp:213-238`), both fail, and the caller gets -`[zip]`. `odr::open` hands back a `zip::ZipFile` — an archive, not a document. +A `.pages` opens as a text document and renders its body text. `.numbers` and +`.key` have `FileType` entries and `file_type_table.cpp` rows so a caller can +name them, but no capabilities and no engine behind them: which app wrote a +package is read off its root archive type, and neither has a fixture to pin +that against. -Two fixtures are already committed: +Two fixtures are committed: `test/data/input/odr-public/pages/{empty.pages,style-various-1.pages}`, both written by iWork 13.2 (`Metadata/BuildVersionHistory.plist`). Neither is listed -in `index.csv` and neither has reference output, so nothing exercises them. -`style-various-1.pages` carries `Index/Tables/` and nine files under `Data/`, -which is most of the surface below. - -New `FileType` entries append at the end of the enum — `file.hpp:98` says so — -and **the bindings do need updating** here, unlike rtf: `python/src/bind_file.cpp`, -`jni/java/app/opendocument/core/FileType.java`, -`apple/include/OdrCoreObjC/ODRFile.h` + `apple/src/ODRFile.mm`. Wasm does not: -it derives its enums from `odr::all_file_types()` at runtime -(`wasm/src/wasm_core.cpp:35`, `:69`). +in `index.csv` — they do not need to be, `TestData` picks up anything the file +type table knows an extension for — and they gained reference output when stage +2 turned `translate_html` on. `style-various-1.pages` carries `Index/Tables/` +and nine files under `Data/`, which is most of the surface below. ## Spec @@ -145,11 +141,26 @@ silently. --- -## Stage 1 — detection and the container +## Stage 1 — detection and the container *(landed)* Nothing renders yet. The point is that the bytes come apart correctly and the type is reported, which is also the whole of what a file picker needs. +Landed as planned, with three deviations: + +- **Only `iwork_pages` detects and opens.** `iwork_numbers` and `iwork_keynote` + are classification-only rows, because the root archive type of a `.numbers` + or a `.key` cannot be read off a fixture that does not exist, and this module + does not guess. +- **`password_encrypted()` is not answered.** `Index/Metadata.iwph` was going + to report it, but nothing here has ever seen an encrypted package; an + encrypted one is one whose `Index/Document.iwa` does not decompress, and it + falls back to being reported as a zip. +- **Detection does not read `Index/Metadata.iwa`.** It reads + `/Index/Document.iwa` straight, since it runs on every zip a caller opens and + `Document` is the one component whose file name never carries a suffix. The + component list is read when the document is. + - `iwork_snappy.{hpp,cpp}` — Apple framing plus block decompression, over the `std::istream *` / `std::streambuf *` shape `pdf::ObjectParser` uses (`pdf_object_parser.hpp`). @@ -183,7 +194,7 @@ type ID that must be skipped rather than thrown on, and framing truncated mid-block. Only the type-reporting test needs the fixtures — the data repos are fetched and optional, so everything that can be inline is. -## Stage 2 — Pages text +## Stage 2 — Pages text *(landed)* - walk from the document archive to the body's text storage (`TSWP.StorageArchive` in the reverse-engineering literature; confirm the type @@ -199,6 +210,13 @@ fetched and optional, so everything that can be inline is. produce an empty body and not an exception. - table row: `iwork_pages` gains `.translate_html = true`. +Landed as planned. What the fixtures settled: the body storage is field 4 of +`TP.DocumentArchive` (type 10000) and is a `TSWP.StorageArchive` (type 2001); +its paragraph style table is field 5, a `TSWP.ObjectAttributeTable` whose +field 1 repeats the entries — so the run tables are one level deeper than +"repeated entries on the storage". Run-table indices count UTF-16 code units +against UTF-8 text, which the parser translates in one pass. + ## Stage 3 — Pages styles - `Index/DocumentStylesheet.iwa`. Style archives are sparse property sets with a @@ -286,10 +304,11 @@ without a Numbers fixture existing. ## Test data -`empty.pages` and `style-various-1.pages` are already in -`test/data/input/odr-public/pages/` but absent from `index.csv` and from -reference output. Add them to the index in stage 1, and regenerate reference -output when stage 2 flips `translate_html` on. +`empty.pages` and `style-various-1.pages` are in +`test/data/input/odr-public/pages/`. They need no `index.csv` row — +`TestData::test_files` picks up any file whose extension the file type table +knows — and reference output was regenerated when stage 2 flipped +`translate_html` on. Stages 5 and 7 each need a fixture that does not exist yet — one `.key` and one `.numbers` in the public repo. Everything at container level stays inline, per From 98c418d4feb774c475db0ab121e7914501ee0cb4 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 23 Aug 2026 18:06:19 +0200 Subject: [PATCH 7/7] test(data): pin the reference output the two pages fixtures render Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GADNSpk1CY88GMpqkafN6z --- test/data.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/data.cmake b/test/data.cmake index ffdcf2fc..26286106 100644 --- a/test/data.cmake +++ b/test/data.cmake @@ -17,7 +17,7 @@ odr_test_data( odr_test_data( PATH "reference-output/odr-public" URL "https://github.com/opendocument-app/OpenDocument.test.output.git" - REVISION "d9cb5666399ab5873152953106dce53723fcbced") + REVISION "f8994d207ce3869e05e4ba126cd9d3de1fc73bc7") odr_test_data( PATH "reference-output/odr-private"