Share parsed files instead of re-parsing every include

include/openscad_cpp_parser/api.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "openscad_cpp_parser/ast.hpp"
+#include "openscad_cpp_parser/scope_table.hpp"
 #include "openscad_cpp_parser/source_map.hpp"
 
 #include <memory>
@@ -68,15 +69,10 @@ std::vector<std::unique_ptr<ASTNode>> getASTFromString(const std::string& code,
 // can't be found/read, or ParseError (see getASTFromString) on a syntax
 // error in any of the involved files.
 //
-// ponytail: unlike the Python reference, this does NOT cache parsed ASTs
-// (in-memory or on-disk) across calls. With unique_ptr ownership, "return
-// the same cached tree to every caller" isn't representable (each caller
-// needs exclusive ownership), and a clone-based cache would need a deep
-// clone() for all 66 node kinds just to support an unmeasured perf
-// optimization. Every call re-parses. Upgrade path: add a per-node
-// clone() (mirroring toJson/toString's per-kind dispatch) and cache
-// serialized snapshots keyed by (path, mtime) if repeated-parse cost of a
-// real workload is shown to matter.
+// This does NOT share or cache anything: every call re-parses the file and
+// every file it includes, and the caller owns all of it. See
+// getProgramFromFile() for the shared, cached form -- the one an evaluator
+// that re-renders the same script should use.
 std::vector<std::unique_ptr<ASTNode>> getASTFromFile(const std::string& file, bool includeComments = false,
                                                       bool processIncludes = true);
 
@@ -95,9 +91,44 @@ LibraryFileResult getASTFromLibraryFile(const std::string& currFile, const std::
 // platform default library dir.
 std::optional<std::string> findLibraryFile(const std::string& currFile, const std::string& libFile);
 
-// No-op: kept for API-shape parity with the Python reference's
-// clear_ast_cache(). See the ponytail note on getASTFromFile() -- this
-// port doesn't cache, so there's nothing to clear.
+// A file's statements with its `include <...>` directives resolved, where
+// each included file's AST is SHARED with every other file that includes
+// it rather than re-parsed.
+//
+// That sharing is the point. Parsing `include <BOSL2/std.scad>` costs
+// ~55ms and is ~82% of evaluating a small BOSL2 script; the library does
+// not change between renders, so re-parsing it every time is the single
+// largest cost in a re-render or a docs build (which renders ~1000
+// examples, each including the same library).
+//
+// Only the per-FILE parses are cached, not a whole resolved program: a
+// file is spliced in at most once per resolution (`visited`), so caching
+// std.scad already-resolved would double its statements in a script that
+// also includes something else depending on it. Re-running the resolution
+// is only pointer pushes.
+struct ParsedProgram {
+    // The flattened statement list, in source order, includes spliced in
+    // where their directives stood. Non-owning: see keepAlive.
+    std::vector<const ASTNode*> nodes;
+    // Holds every AST the above points into -- this file's own parse and
+    // each shared include -- alive for as long as this object. Nodes are
+    // borrowed, never owned, which is what lets two scripts (or two
+    // threads) use the same parsed library at once.
+    std::vector<std::shared_ptr<const std::vector<std::unique_ptr<ASTNode>>>> keepAlive;
+};
+
+// Parses `file` and resolves its includes against the cache, which is keyed
+// by (path, mtime, size) so an edited file re-parses on its own.
+// Thread-safe. Throws exactly as getASTFromFile does.
+ParsedProgram getProgramFromFile(const std::string& file, bool includeComments = false);
+
+// Drops every cached file parse. Nothing already handed out is
+// invalidated -- a ParsedProgram keeps what it borrowed alive.
 void clearAstCache();
 
+// How many file parses the cache is holding. For tests that need to prove
+// the cache replaces an edited file's entry rather than accumulating one
+// per save.
+size_t astCacheSize();
+
 } // namespace oscad

include/openscad_cpp_parser/ast.hpp

@@ -7,6 +7,7 @@
 #include "openscad_cpp_parser/ast/expression.hpp"
 #include "openscad_cpp_parser/ast/module_instantiation.hpp"
 #include "openscad_cpp_parser/ast/scope_builder.hpp"
+#include "openscad_cpp_parser/scope_table.hpp"
 #include "openscad_cpp_parser/ast/vector_element.hpp"
 #include "openscad_cpp_parser/position.hpp"
 #include "openscad_cpp_parser/scope.hpp"

include/openscad_cpp_parser/ast/ast_node.hpp

@@ -2,6 +2,7 @@
 
 #include "openscad_cpp_parser/position.hpp"
 
+#include <cstdint>
 #include <string>
 
 namespace oscad {
@@ -98,9 +99,67 @@ const char* nodeKindName(NodeKind kind);
 // Base class for all AST nodes. Mirrors openscad_lalr_parser.nodes.ASTNode:
 // every node carries its source Position and (once buildScope() has run) a
 // non-owning pointer to the Scope visible at that point in the tree.
+// Numbering context for the nodes one parse creates: every node built
+// while it is installed gets that parse's treeId and a slot dense within
+// it -- see ASTNode::slot().
+struct NodeNumbering {
+    uint32_t treeId = 0;
+    uint32_t next = 0;
+};
+
+// Installs a numbering for a whole parse, INCLUDING any pass that builds
+// more nodes from its result (attachComments wraps expressions in
+// CommentedExpr nodes after the parser proper has finished). Nested use
+// reuses the outer numbering rather than starting a second one, so a tree
+// stays a tree however many passes contribute to it.
+//
+// Thread-local: two threads may parse at once and must number
+// independently.
+class ParseNumberingScope {
+public:
+    ParseNumberingScope();
+    ~ParseNumberingScope();
+    ParseNumberingScope(const ParseNumberingScope&) = delete;
+    ParseNumberingScope& operator=(const ParseNumberingScope&) = delete;
+
+private:
+    NodeNumbering numbering_;
+    NodeNumbering* previous_;
+    bool installed_;
+};
+
+class ScopeTable;
+
+// The table setScope() writes into. buildScopes() installs one for its
+// walk; thread-local, so two evaluations can build scopes at once over the
+// same shared tree -- the whole point of sharing it.
+class ScopeTableScope {
+public:
+    explicit ScopeTableScope(ScopeTable& table);
+    ~ScopeTableScope();
+    ScopeTableScope(const ScopeTableScope&) = delete;
+    ScopeTableScope& operator=(const ScopeTableScope&) = delete;
+
+private:
+    ScopeTable* previous_;
+};
+
+NodeNumbering* currentNodeNumbering();
+// Slot for a node built with no parse active -- a test constructing nodes
+// by hand, or astFromJson. Those all land in treeId 0 with process-unique
+// slots, so they never collide with each other.
+uint32_t nextLooseSlot();
+
 class ASTNode {
 public:
-    ASTNode(NodeKind kind, Position position) : kind_(kind), position_(std::move(position)) {}
+    ASTNode(NodeKind kind, Position position) : kind_(kind), position_(std::move(position)) {
+        if (NodeNumbering* n = currentNodeNumbering()) {
+            treeId_ = n->treeId;
+            slot_ = n->next++;
+        } else {
+            slot_ = nextLooseSlot();
+        }
+    }
     virtual ~ASTNode() = default;
 
     ASTNode(const ASTNode&) = delete;
@@ -109,35 +168,46 @@ class ASTNode {
     NodeKind kind() const { return kind_; }
     const Position& position() const { return position_; }
 
-    // Overloaded on the constness of `this` rather than a single `const`
-    // method returning `Scope*` unconditionally: a plain raw-pointer
-    // return type doesn't propagate constness to the pointee, so a caller
-    // holding only `const ASTNode&` (e.g. a read-only borrow shared across
-    // threads while one owner keeps the tree alive) could otherwise still
-    // reach through scope() and call a mutating method like
-    // defineVariable() on the Scope -- defeating the whole point of the
-    // borrow being const. This makes that a compile error instead.
-    Scope* scope() { return scope_; }
-    const Scope* scope() const { return scope_; }
+    // Identity of the parse that built this node, and this node's dense
+    // index within it. Together they address the node's Scope in a
+    // ScopeTable, which is what lets one parsed tree be shared by several
+    // evaluations at once: the scope a node sits in depends on the file
+    // that included it, so it cannot live in the node itself. Both are 0
+    // for a node built outside a parse (a test constructing nodes by
+    // hand), which is a valid single tree of its own.
+    uint32_t treeId() const { return treeId_; }
+    uint32_t slot() const { return slot_; }
+
+    // A node's Scope is NOT stored here -- it lives in the render's
+    // ScopeTable, addressed by (treeId, slot). It has to: one parsed tree
+    // is shared by every script that includes it, and `include` means the
+    // included nodes sit in the INCLUDER's scope, so the same node is in a
+    // different scope in each. Read it with ScopeTable::get(node).
 
     // Mirrors Python's __str__: every leaf node overrides this.
     virtual std::string toString() const = 0;
 
     // Mirrors Python's build_scope(parent_scope): the default (leaf) case
     // just records parent_scope; nodes that introduce bindings or new
     // scopes override this.
-    virtual void buildScope(Scope& parentScope) { scope_ = &parentScope; }
+    virtual void buildScope(Scope& parentScope) { setScope(parentScope); }
 
 protected:
     // For buildScope() overrides: records the scope visible at this node,
     // mirroring Python's `self.scope = parent_scope` (which is often a
     // *different* scope than what gets passed to this node's children).
-    void setScope(Scope& s) { scope_ = &s; }
+    //
+    // Writes into the ScopeTable buildScopes() installed for the duration
+    // of its walk, rather than taking one as a parameter: that keeps all
+    // 37 buildScope() overrides on their existing signature, and the walk
+    // is a single scoped pass with nothing else running inside it.
+    void setScope(Scope& s);
 
 private:
     NodeKind kind_;
     Position position_;
-    Scope* scope_ = nullptr;
+    uint32_t treeId_ = 0;
+    uint32_t slot_ = 0;
 };
 
 } // namespace oscad

include/openscad_cpp_parser/scope.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "openscad_cpp_parser/scope_table.hpp"
+
 #include <memory>
 #include <string>
 #include <unordered_map>
@@ -83,6 +85,19 @@ class Scope {
         return *children_.back();
     }
 
+    // The ScopeTable holding every node's scope for this scope tree.
+    // buildScopes() attaches it to the root it returns, so the table lives
+    // exactly as long as the scopes it points at -- which is the ownership
+    // contract callers already keep. Null on a non-root scope; use
+    // rootTable() to reach it from anywhere in the tree.
+    void adoptTable(std::unique_ptr<ScopeTable> table);
+    const ScopeTable* table() const { return ownedTable_.get(); }
+    const ScopeTable* rootTable() const {
+        const Scope* s = this;
+        while (s->parent_) s = s->parent_;
+        return s->ownedTable_.get();
+    }
+
 private:
     static ASTNode* find(const std::unordered_map<std::string, ASTNode*>& table, const std::string& name) {
         auto it = table.find(name);
@@ -94,6 +109,16 @@ class Scope {
     std::unordered_map<std::string, ASTNode*> functions_;
     std::unordered_map<std::string, ASTNode*> modules_;
     std::vector<std::unique_ptr<Scope>> children_;
+    std::unique_ptr<ScopeTable> ownedTable_;   // root only
 };
 
+// The lexical Scope recorded for `node` when `anyScopeInTree`'s tree was
+// built -- the replacement for the ASTNode::scope() field that used to
+// live in the node. Any scope in the tree will do; it walks up to the root
+// that owns the table.
+inline const Scope* scopeOf(const Scope& anyScopeInTree, const ASTNode& node) {
+    const ScopeTable* table = anyScopeInTree.rootTable();
+    return table ? table->get(node) : nullptr;
+}
+
 } // namespace oscad

include/openscad_cpp_parser/scope_table.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "openscad_cpp_parser/ast/ast_node.hpp"
+
+#include <cstdint>
+#include <vector>
+
+namespace oscad {
+
+class Scope;
+
+// Where a node's Scope lives, now that it cannot live in the node.
+//
+// A parsed tree is shared: `include <BOSL2/std.scad>` is parsed once and
+// then handed to every script that includes it, which is what makes a
+// render cheap (parsing BOSL2 is ~55ms and dwarfs everything else). But
+// `include` means "share my scope", so the very same BOSL2 node sits in a
+// different scope in every script that includes it. A `Scope*` field in
+// ASTNode could only ever hold one of them, and whichever render wrote it
+// last would corrupt the others.
+//
+// So the pointer moves out here, into a table one render owns. Addressing
+// is (treeId, slot), both stamped on the node when it was parsed, so a
+// lookup is two loads and no hashing -- it sits on the VM's call path.
+class ScopeTable {
+public:
+    // Null until set: buildScopes() fills in every node it walks, and a
+    // reader treats null as "no scope recorded", exactly as the old
+    // nullptr-initialised field did.
+    const Scope* get(const ASTNode& node) const {
+        const uint32_t tree = node.treeId();
+        if (tree >= trees_.size()) return nullptr;
+        const std::vector<Scope*>& slots = trees_[tree];
+        const uint32_t slot = node.slot();
+        return slot < slots.size() ? slots[slot] : nullptr;
+    }
+    Scope* get(const ASTNode& node) {
+        return const_cast<Scope*>(static_cast<const ScopeTable*>(this)->get(node));
+    }
+
+    void set(const ASTNode& node, Scope* scope) {
+        const uint32_t tree = node.treeId();
+        if (tree >= trees_.size()) trees_.resize(tree + 1);
+        std::vector<Scope*>& slots = trees_[tree];
+        const uint32_t slot = node.slot();
+        if (slot >= slots.size()) slots.resize(slot + 1, nullptr);
+        slots[slot] = scope;
+    }
+
+private:
+    // Indexed by treeId. Sparse in principle -- treeIds are handed out
+    // process-wide and a render only involves a few trees -- but the empty
+    // outer entries are 24 bytes each and never allocate, so the waste is
+    // a few tens of KB against a 55ms saving.
+    std::vector<std::vector<Scope*>> trees_;
+};
+
+} // namespace oscad