diff --git a/docs/hermitcrab-forest-memo-ceiling.md b/docs/hermitcrab-forest-memo-ceiling.md new file mode 100644 index 00000000..23df588e --- /dev/null +++ b/docs/hermitcrab-forest-memo-ceiling.md @@ -0,0 +1,180 @@ +# Forest memoization: the most we think we can get + +> **OUTCOME (2026-08-26): the ceiling was never approached, because R came in at 1.12 on Sena +> and 1.17 on Indonesian against a 1.3 gate.** Section 6's falsification table fired on its first +> row. The measured numbers and the analysis of why the worst case collapses while the realised +> state count does not are in `hermitcrab-forest-memo-plan.md` sections 3.1–3.5. The rest of this +> file is left exactly as written *before* the measurement, because its value now is as a record +> of what was predicted and how the prediction did. +> +> How the predictions scored, for calibration: Indonesian was predicted 1.0–1.3 and came in at +> 1.17 — right. Sena was predicted 2–4 and came in at 1.12 — wrong, and wrong in the direction +> that decides the project. Amharic was predicted "most likely to disappoint" and had by far the +> best static classification (31 of 36 rules provably shrinking) — wrong again, though its +> realised R is what actually matters. + + +Companion to `hermitcrab-packed-forest-research.md` (why) and +`hermitcrab-forest-memo-plan.md` (how). This file answers one question only: **if everything +in the plan works, what is the number?** + +Written 2026-08-26 on `feature/forest-memo`. Every input is cited. Where a figure is derived +rather than measured, it says so — Stage 0 of the plan replaces the derived ones with measured +ones, and this file should be rewritten once it does. + +--- + +## 1. The budget we are spending against + +Sena `atawirambo`, the reference heavy word, on the sequential+memo path that +`feature/memoization` ships: + +| | measured | +| --- | --- | +| fair sequential unmemoized baseline | 30.5 s | +| after mrule memo + template memo (current HEAD) | **6.1 s** | +| morphological-rule cascade, memoized | 1.4 s / 2,555 expansions vs a 2,546-state floor | +| affix-template battery, unmemoized | 93% of the 30.5 s, run 38,840x | +| distinct `AnalysisStateKey` values | ~2,581 | + +**Derived, not measured:** the template battery costs about 28.4 s over 38,840 runs, so roughly +0.73 ms per run. Memoized, it runs once per distinct key — about 2,581 times — for roughly +**1.9 s**. Adding the cascade's 1.4 s, about **3.3 s of the current 6.1 s scales with the +distinct-key count.** The residual **~2.8 s** is lexical lookup, `ReplayOnto` materialization, +`ExpandAlternatives`, and forward synthesis. + +That split is the whole ceiling argument, and it is the first thing Stage 0 must confirm +directly. If the real split is 1.5 s / 4.6 s, every number below halves. + +--- + +## 2. Bound 1 — key narrowing + +Dropping shrinking-rule counts from `AnalysisStateKey` merges states that differ only in *which +shrinking affixes have been stripped along the way*. Call the resulting state-count reduction +**R**. + +Both key-proportional subsystems scale by 1/R, because the mrule memo and the template memo are +keyed by the same object and each runs once per distinct key. So: + +| R (state-count reduction) | key-proportional cost | word total | speedup | +| --- | --- | --- | --- | +| 1.0 (no collapse — the change is inert) | 3.3 s | 6.1 s | 1.00x | +| 1.5 | 2.2 s | 5.0 s | 1.22x | +| 2 | 1.65 s | 4.45 s | **1.37x** | +| 4 | 0.83 s | 3.6 s | 1.68x | +| infinite (free) | 0 s | 2.8 s | **2.18x** | + +**The asymptote is 2.2x on this word, and no amount of cleverness in the key gets past it.** +That is the honest ceiling for the proposal as stated. + +### What sets R + +R is driven by **affix homophony**. Two paths only collide after narrowing if they stripped +*different* affix sets and arrived at the same shape with the same feature structures — which +requires distinct affixes with the same surface form and compatible feature effects. Sena has +many; Indonesian has few. This is why the plan measures R before building anything. + +Our prior on R, stated so it can be scored later: **Sena 2–4, Indonesian 1.0–1.3, Amharic +unknown.** If Sena comes back at 1.1 the whole line of work is inert and Stage 1 should not be +built. + +### The sleeper effect + +Narrowing the key also raises the **nogood** hit rate — `MemoEntry` with an empty `Results` list +short-circuits an entire subtree, and there are more ways to hit a narrower key. This is free +upside not modelled in the table above, and it is worth counting separately in Stage 0 because +nogood hits are the cheapest possible win. + +--- + +## 3. Bound 2 — the forest proper + +Back-edges plus deferred materialization attack the ~2.8 s residue, in two ways: + +**Per-state lexical lookup instead of per-path.** Today `LexicalLookup` runs once per analysis +candidate. With a forest, the shape is a property of the state, so it can run once per state. +`atawirambo`: 41 candidates reach the lexicon check, 4 reach the lexicon, 2 parse. The T1 probe +bounds what this can prune on the words that actually hurt: **pooled 23.5% of steps are +lexically dead on failure words**, and `pidafikawo` is at exactly 0.0% — a root substring exists +at every node visited and the word still fails, on checks that only run *after* lexical lookup +succeeds. + +**Less materialization.** `ReplayOnto` clones every stored result on every hit. Deferring +materialization to the paths that actually reach synthesis removes that. Calibration from the +allocation work: shape sharing at the two proven-safe sites bought **-4.5 to -7.6% bytes** with +no wall-clock change, and pooling small collections was a **net loss on every axis**. Allocation +reduction on this codebase has consistently converted to wall clock at well under 1:1. + +**Realistic contribution: 10–20% of the residue, i.e. 0.3–0.6 s.** Combined with Bound 1 at +R=2, that is roughly **4.0 s, or 1.5x**. At R=4, roughly **3.1 s, or 2.0x**. + +--- + +## 4. What this does *not* touch — and why the failure words are the real prize + +`atawirambo` succeeds. The words that cost the most fail: + +| word | steps | synthesis inputs | parses | +| --- | --- | --- | --- | +| `atawirambo` | 14.9 M | — | 2 | +| `cinacemerwa` | 37.5 M | **218,847** | **0** | + +For `cinacemerwa` (26.9 s post-memo) the cost is not state expansion at all. It is 218,847 +forward synthesis runs that all fail. Bounds 1 and 2 barely touch it: narrowing the key does not +reduce the number of *paths*, only the number of *states*, and the synthesis input set is +path-shaped. + +**The only lever on that number is merging order variants at readout**, which requires proving +rule-pair commutativity — and which we have measured as **unsound to assume**: same pending-rule +multiset, different order, different synthesis output, on Sena and independently on Indonesian. +See the research doc, section 2.2. + +If a static or verify-once commutativity analysis lands, the F1 probe's **28.72x** aggregate +synthesis-input dedup on Sena heavy words becomes reachable, and on a word like `cinacemerwa` +that is close to the whole runtime. **That, not the key narrowing, is where an order of +magnitude lives.** It is out of scope for this branch and is written up as the follow-on. + +--- + +## 5. The honest headline + +**What we expect to be able to claim at the end of this branch:** + +- Sena heavy words: **1.3–2.0x**, contingent on R landing in the 2–4 range. +- Indonesian: **1.0–1.1x**. Its state pressure is low and its F1 dedup ratio was 1.41x. This + change is not aimed at Indonesian and should not be sold as if it were. +- Amharic: unknown, must be measured, and it is the grammar most likely to surprise us — it has + infixation, truncation-reinsertion, and a `ModifyFromInput`, so its shrinking-rule set may be + small and R may be near 1. +- A **worst-case key-space reduction from 2^k to 2^k'**, where k' counts only the + non-shrinking rules. For a template-only grammar k' is 0 and the key becomes + (shape, stratum, feature structures) — which is the correspondent's claim, and it is correct + for that restricted class. +- A **clean termination proof** for the memo, replacing "we count every rule because some rule + might loop" with "we count exactly the rules that can loop." + +**What we do not expect to be able to claim:** + +- Polynomial end-to-end parsing. Analysis becomes polynomial in input length for the restricted + class; the grammar constant is a feature-structure lattice, and synthesis remains + path-enumerated. Maxwell & Kaplan 1993 is explicit that the exponential lives at the interface + between the packable component and the constraint component, not inside either. +- Any material improvement on the pathological failure words. That needs commutativity. + +**The single number to lead with, if one is wanted: about 2.2x is the asymptote on our reference +heavy word, and we expect to realise somewhere between a half and all of it.** + +--- + +## 6. Falsification conditions + +Written before the measurements, so they cannot be adjusted afterwards. + +| If Stage 0 shows | then | +| --- | --- | +| R < 1.3 on Sena | stop. Do not build Stage 1. Report the negative result. | +| key-proportional share < 30% of wall time | the ceiling table is wrong; recompute before building | +| readout-time count filtering raises synthesis inputs by >10% | Stage 2 is a net loss; keep the counts in the key and take only the forest | +| Amharic R >> Sena R | the change is more general than we thought; raise ambition | +| any corpus shows an analysis-set difference | stop and fix; completeness is not negotiable | diff --git a/docs/hermitcrab-forest-memo-plan.md b/docs/hermitcrab-forest-memo-plan.md new file mode 100644 index 00000000..7536990a --- /dev/null +++ b/docs/hermitcrab-forest-memo-plan.md @@ -0,0 +1,397 @@ +# Forest memoization: implementation plan + +Branch: `feature/forest-memo`, off `feature/memoization` (`af809180`). +Companions: `hermitcrab-packed-forest-research.md` (why), `hermitcrab-forest-memo-ceiling.md` +(what the number can be). Section 8 of this file is the PR description, ready to lift. + +Baseline on this branch at creation: **82/82 HermitCrab tests green**, Release, net10.0. + +--- + +## 1. The change in one paragraph + +`AnalysisStateKey` currently carries a per-rule unapplication-count multiset covering *every* +morphological rule. Because `MaxApplicationCount` defaults to 1, that component degenerates into +"which subset of the grammar's rules has been unapplied so far" — 2^k values in the worst case, +all sharing one shape and one pair of feature structures. Most of those distinctions are not +load-bearing: a rule whose unapplication *shrinks* the word cannot drive an infinite regress, +because the shape length is already a decreasing measure. This plan classifies rules by their +length effect under unapplication, keeps only the non-shrinking ones in the key, moves the +per-rule count limit from the search to a post-analysis filter, and records `` back-edges so derivations remain recoverable. Termination is preserved by a lexicographic +measure: shape length strictly decreases along a shrinking edge, and the retained-rule count +strictly increases along a non-shrinking one, so no cycle can close. + +--- + +## 2. Safety invariants + +These are not negotiable and every stage is checked against them. + +1. **Search completeness is never reduced.** HermitCrab is the permanent fallback engine behind + the FST work. A faster parser that loses parses is not a faster parser. +2. **Conservative default in the classifier.** *Keeping* a rule in the key is always sound — it + is the status quo. *Dropping* one is the risky direction. Any rule the classifier cannot + prove strictly shrinking is retained. `Unknown` is not a failure mode, it is the safe answer. +3. **The acceptance gate is analysis-set equality, not byte equality.** Compare canonical + morpheme-signature sets (`join("+", morphemeIds)` plus root index, sorted, semicolon-joined), + never object or byte identity. A replayed `Word` is legitimately not field-for-field + identical to a freshly computed one. +4. **Never couple a pruning gate to memo presence.** The prototype bundled Phase 5's + `HasReachableRoot` into memo-on and the Rust port had to remove it. Whatever the key does, + memo-on and memo-off must return the same set. +5. **Freeze on read in the key constructor.** `AnalysisAffixTemplateRule.Apply` reassigns + `SyntacticFeatureStruct` to a fresh unfrozen clone after the owning `Word` is frozen, and + that setter has no `CheckFrozen()` guard. The existing defensive `Freeze()` calls in + `AnalysisStateKey`'s constructor stay. + +--- + +## 3. Stage 0 — measure before building (no product code) + +**This stage can kill the whole plan, and that is its job.** Nothing in +`src/SIL.Machine.Morphology.HermitCrab` changes. + +Add a diagnostic in the test assembly, modelled on the existing +`tests/.../MemoCorpusVerification.cs` (explicit-category, corpus-gated, skipped when the +grammar files are absent). It reports, per grammar and per heavy word: + +| metric | why | +| --- | --- | +| rule classification census (shrinking / non-shrinking / unknown, per stratum) | is there anything to drop? | +| distinct full keys vs distinct narrowed keys = **R** | the entire Bound-1 ceiling | +| nogood hits under each key | free upside, counted separately | +| wall-time split: mrule cascade / template battery / lexical lookup / synthesis | confirms or destroys the ceiling doc's derived 3.3 s / 2.8 s split | +| synthesis input count (`ExpandAlternatives` outputs reaching `_synthesisRule.Apply`) | the number that must not inflate in Stage 2 | + +Word sets: Sena heavy words (`atawirambo`, `cinacemerwa`, `kukucitirani`, `manyeredzero`, +`pidafikawo`, `cinagumanika`, `kamatamisa`) plus the first 300; Indonesian all 121; Amharic a +bounded head of the corpus — the full Amharic run took about 4.3 hours last time and word 29 +(`ሌባዎቹ`) alone is pathological, so use `--start`-style resumption or a subset, and say which. + +Instrumentation trick that already works here: an instrumented clone swapped in via reflection +plus the tests-assembly `InternalsVisibleTo`, as `HcDissect` did. + +**Gates (from the ceiling doc's falsification table):** + +- R ≥ 1.3 on Sena, else **stop and report the negative result**. +- key-proportional share ≥ 30% of wall time, else recompute the ceiling before building. +- Record R for all three grammars regardless — Amharic is the one most likely to surprise, since + infixation, truncation-reinsertion and a `ModifyFromInput` may leave it with almost no + provably-shrinking rules. + +Deliverable: a committed results table in this file, replacing the ceiling doc's derived +figures with measured ones. + +### 3.1 Results — rule classification census + +`ForestMemoCensus.RuleClassificationCensus`, run 2026-08-26 against the local grammars. + +| grammar | rules | Shrinking | NonShrinking | Unknown | retained in key | worst-case key subsets | +| --- | --- | --- | --- | --- | --- | --- | +| Sena | 27 | 19 | 0 | 8 | **8** | 2^27 -> 2^8 | +| Indonesian | 15 | 10 | 0 | 5 | **5** | 2^15 -> 2^5 | +| Amharic | 36 | 31 | 4 | 1 | **5** | 2^36 -> 2^5 | + +The classifier proves the majority of rules shrinking on all three grammars, so the worst-case +key space collapses hard everywhere. Amharic — predicted in the ceiling doc as the grammar most +likely to have almost no provably-shrinking rules — is in fact the best case, 31 of 36. Its 4 +`NonShrinking` verdicts are the only genuine zero-or-truncating rules in any of the three; Sena's +and Indonesian's retained rules are all `Unknown` (compounding and reduplication), which is the +classifier being conservative rather than the grammar being awkward. + +That is the worst case. What matters is the *realised* collapse, below. + +### 3.2 Results — realised key collapse (R) + +| grammar | words | state-weighted R | words at or above the 1.3 gate | +| --- | --- | --- | --- | +| Indonesian | 120 | **1.17** | 3/120 | +| Sena | 7 heavy words | **1.12** | 1/7 | +| Amharic | 24 words (corpus head) | 1.36 | 1/24 | + +**The Amharic 1.36 does not clear anything, and must not be quoted as if it did.** It is a +state-weighted pooled average over words whose state counts are trivial — the largest is 212 +distinct keys, against Sena's 18,686 — and 40% of the numerator comes from one word (`ለካ`, 198 -> +73 keys, R = 2.71). Every other word sits between 1.00 and 1.25. The subset also deliberately +stops before word 29 (`ሌባዎቹ`), the one genuinely pathological Amharic word, so the grammar's +actual worst case is unmeasured. Pooling a ratio over words of wildly different size is exactly +the error the T1 tandem probe made and had to retract; it is flagged here so it is not made a +second time. + +What Amharic does show, more usefully: a *consistent* small collapse (1.03–1.25 nearly +everywhere) rather than Sena's flat 1.00. That fits the census — Amharic is the only one of the +three with genuine `NonShrinking` rules, so more rules get dropped and states merge steadily but +slightly. + +And one striking corroboration of the central finding: `ሁለተኛ` takes 15,994 ms with 124 states, +and `ሄዶ` takes 30,335 ms with 212 states and 186 synthesis inputs. **State count and cost are +decoupled by two orders of magnitude on this grammar.** Roughly 160 ms per synthesis run is where +Amharic's time actually goes. Nothing done to the analysis state key can touch that. + +Sena, per word: + +| word | ms | parses | key builds | full keys | narrowed keys | R | memo hits | nogood hits | template hits | synthesis inputs | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| atawirambo | 16,709 | 2 | 159,557 | 2,556 | 2,556 | **1.00** | 29,736 | 88,426 | 37,512 | 17,699 | +| kamatamisa | 39,199 | 4 | 421,889 | 13,915 | 13,914 | **1.00** | 20,505 | 340,294 | 43,013 | 44,377 | +| manyeredzero | 14,933 | 0 | 96,023 | 11,762 | 11,762 | **1.00** | 2,794 | 73,460 | 5,812 | 1,074 | +| pidafikawo | 14,755 | 0 | 117,022 | 3,700 | 3,262 | 1.13 | 14,113 | 75,580 | 21,882 | 5,016 | +| kukucitirani | 72,788 | 5 | 541,712 | 18,130 | 18,130 | **1.00** | 48,486 | 415,718 | 53,407 | 158,480 | +| cinagumanika | 38,431 | 0 | 305,986 | 11,896 | 10,289 | 1.16 | 22,920 | 231,677 | 35,416 | 47,517 | +| cinacemerwa | 74,075 | 0 | 528,057 | 18,686 | 13,588 | **1.38** | 25,102 | 434,628 | 45,359 | 218,847 | + +**The gate is not met. R = 1.12 on Sena against a 1.3 bar, and 6 of 7 words show no collapse at +all.** + +The harness is measuring the right thing — two independent cross-checks say so. `atawirambo` +reports 2,556 distinct full keys against the 2,546-state floor measured on +`parse-optimization` by completely different instrumentation, and `cinacemerwa` reports +218,847 synthesis inputs, matching the historical figure exactly. + +### 3.3 Why the worst case collapses and the realised state count does not + +The census says the key space drops from 2^27 to 2^8 on Sena. The measurement says the number +of states actually visited drops by 0%. Both are true, and the gap is the whole finding. + +For two paths to merge once the shrinking counts are dropped, they have to strip *different* +affix sets and still land on the same shape with the same syntactic and realizational feature +structures. The shape and the feature structures already discriminate almost perfectly: a +different affix set almost always means a different residue or a different set of required head +features. The 2^k blowup the proposal identifies is real as a bound and essentially never +realised on these grammars — the rule-count component of the key was, in practice, already +implied by the components next to it. + +This is the same lesson as "counting redundant expansions is not counting cost", in a new +guise: **a worst-case bound collapsing is not the same as the realised state count collapsing.** + +### 3.4 What the data says instead + +Three observations worth keeping, none of which were what we set out to measure. + +**R tracks failure, not size.** Every word with R = 1.00 has parses (`atawirambo` 2, +`kamatamisa` 4, `kukucitirani` 5) or is cheap (`manyeredzero`). Every word with R > 1 returns +zero parses. The collapse happens exactly where the search wanders into territory that leads +nowhere — which is at least the right place for it to happen. `cinacemerwa`, the single most +expensive word in the corpus, is the best case at 1.38. That is not enough to carry the change, +but it is the opposite of noise. + +**The nogood cache is carrying the memo, by an order of magnitude.** On `cinacemerwa`, 434,628 +nogood hits against 25,102 positive replays; on `kukucitirani`, 415,718 against 48,486. The +expensive part of analysis is proving subtrees empty, not reusing subtrees that produced +something. Any future work here should be aimed at the nogood path. + +**Key construction is hit about 28x per distinct state** (`cinacemerwa`: 528,057 builds for +18,686 states). The memo is doing a great deal of work and the state count is a genuine floor. + +### 3.5 Verdict + +Per the ceiling doc's falsification table, written before the measurement: *"If Stage 0 shows +R < 1.3 on Sena, stop. Do not build Stage 1's consumers. Report the negative result."* + +**Stopping.** Stages 2 through 4 are not built. The classifier and the census harness stay — +they are the evidence, they are correct, and they are reusable by anything that needs to know +which rules can grow a word. The Stage 2 wiring exists as a stash on this branch and should not +be merged on the strength of a 1.12. + +What would change this verdict: a grammar whose R is genuinely high. The classification census +is cheap to run and is the right first question to ask of any new grammar — but on the three +grammars this project has, the answer is no. + +--- + +## 4. Stage 1 — the length-effect classifier + +New file `src/SIL.Machine.Morphology.HermitCrab/RuleLengthEffect.cs`, or an addition to a ported +`GrammarAnalyzer` (the archive version on `parse-optimization-archive` has the reusable +`AffixProcessAllomorph.Rhs` walk and the hard-won doc comments; port only what is used). + +``` +internal enum UnapplicationLengthEffect { Shrinking, NonShrinking, Unknown } +internal static UnapplicationLengthEffect Classify(IMorphologicalRule rule) +``` + +Classification rules: + +- **`AffixProcessRule` / `RealizationalAffixProcessRule`** — unapplication removes what the + allomorph's `Rhs` inserts and preserves what it copies. + - every allomorph inserts at least one segment (`InsertSegments` / `InsertSimpleContext`), + copies each part at most once, and has no part copied twice -> **Shrinking** + - any allomorph inserts nothing (a zero morpheme) -> **NonShrinking** + - reduplication (a part copied more than once), or anything the `Rhs` walk does not recognise + -> **Unknown** +- **`CompoundingRule`** -> **Unknown**. Unapplication splits a word into head and non-head; + total material is preserved even though the head shrinks. `NonHeadCount` is in the key + independently, so there is nothing to gain by being clever here. +- **Anything else** -> **Unknown**. + +`Unknown` and `NonShrinking` are treated identically by the key (both retained). They are kept +distinct in the enum so the Stage 0 census can tell "this grammar has zero morphemes" apart from +"this grammar has constructs we cannot analyse." + +Tests (`RuleLengthEffectTests.cs`), all on hand-built grammars: + +- ordinary suffix -> Shrinking +- zero morpheme -> NonShrinking +- reduplication -> Unknown +- infixation -> Shrinking if it still inserts material (it does) — assert the direction + explicitly, because this is the case most likely to be got backwards +- compounding rule -> Unknown +- a rule with a mix of inserting and zero allomorphs -> NonShrinking (the weakest allomorph + governs) + +No behaviour change. Classification only, plus the Stage 0 census consuming it. + +**Exit criterion:** classifier tests green, census numbers committed, 82/82 still green. + +--- + +## 5. Stage 2 — narrow the key and move the count gate + +These two sub-parts are unsound apart and must land in one commit. + +**2a. Narrow the key.** `AnalysisStateKey` filters `word.UnappliedRuleCounts` to entries whose +rule classifies as `NonShrinking` or `Unknown`. For a template-only grammar the filtered +multiset is empty and the key collapses to `(Shape, Stratum, SyntacticFS, RealizationalFS, +NonHeadCount)` — which is exactly the correspondent's claim, and correct for that class. + +Do the filtering once per `Morpher`, not once per key construction: build an +`IReadOnlyDictionary retainInKey` at grammar-load time and hang it off +`AnalysisScope`. Key construction is on the hottest path in the engine; a per-key LINQ filter +would eat the win it is trying to create. + +**2b. Move the count limit to readout.** `AnalysisAffixProcessRule.cs:45` and +`AnalysisCompoundingRule.cs:46` stop enforcing `MaxApplicationCount` during analysis *for rules +classified Shrinking* — they must, because once a shrinking rule leaves the key, two words with +different counts for it share a memo entry and the gate would otherwise give them different +answers. A post-analysis filter in `Morpher.ParseWord`, sitting between `_analysisRule.Apply` +(`Morpher.cs:141`) and `Synthesize` (`Morpher.cs:155`), drops any candidate whose trail exceeds +any rule's `MaxApplicationCount`. + +Note that `SynthesisAffixProcessRule.cs:46` already enforces the same limit on the way back, so +an over-count derivation cannot escape into the results even if the new filter is wrong — but it +would escape into *synthesis*, which is the expensive phase. The explicit filter is what keeps +Stage 2 from being a net loss, and its effectiveness is measured directly as the +synthesis-input count from Stage 0. + +**Rollout:** `Morpher.NarrowAnalysisStateKey`, default `true`, meaningful only when the memo path +is active (`maxDegreeOfParallelism: 1`). The flag exists so the A/B harness can flip it inside +one process; it is not a user-facing feature. + +Tests: + +- **the correspondent's own cycle fixture**: a grammar with a zero morpheme turning N into V and + another turning V into N. Assert the parse terminates and returns the same set as memo-off. + This is the case the `>=` boundary exists for, and it belongs in the suite by name. +- key equality: two words differing only in shrinking-rule counts -> equal keys; differing in a + zero-morpheme count -> unequal keys. +- count limit: a rule with `MaxApplicationCount = 2` and a word admitting three unapplications -> + the three-unapplication analysis is absent from the result set, and the set equals memo-off. +- the `SelfOpaquing` two-iteration simultaneous-epenthesis fixture from the memoization plan — + PanGloss flagged a real latent C# nogood-cache divergence in this exact code path (memo-on 0 + parses vs memo-off 1) whose trigger was never isolated. If it reproduces here, fix it and draft + a JIRA issue per the standing C#-oracle-bug process. +- `DiagMemoHits` / `DiagNogoodHits` / `DiagTemplateMemoHits` asserted non-zero wherever a test + covers the replay path, so a memo that silently stopped firing cannot look like a pass. + +**Gates:** analysis-set identical against this branch's HEAD, memo-on, on Sena (first 300 + +the heavy set), Indonesian 121/121, Amharic (same subset Stage 0 used). Synthesis input count +must not rise more than 10%. 82/82 plus the new tests green in both flag states. + +--- + +## 6. Stage 3 — back-edges, written but not read + +The forest proper, added in a shape that cannot break anything because nothing depends on it yet. + +Record on each memo entry the predecessor edges `(AnalysisStateKey source, IMorphologicalRule +rule)` for every successful rule application that produced it — the `` pair +from the proposal. Store them in a new `AnalysisScope.Forest` table rather than widening +`MemoEntry`, so the existing replay path is untouched. + +Then a test-only verifier enumerates derivations from the forest and asserts the enumerated set +equals the materialized result set, on real corpus words. This is the cheap way to prove the +representation correct before anything depends on it — and it is the stage that would catch a +mistaken termination argument, because a cycle in the forest shows up here as a hang or a +duplicate rather than as a wrong answer in production. + +**Gate:** forest enumeration equals materialization on all three grammars; no wall-clock +regression beyond noise (the forest is write-only at this stage, so any regression is pure +bookkeeping cost and must be small enough to carry into Stage 4). + +--- + +## 7. Stage 4 — deferred materialization (optional, gated) + +Only if Stages 0 and 3 justify it. The two wins, from the ceiling doc: + +- **per-state lexical lookup** instead of per-path — bounded by the T1 probe's pooled 23.5% + lexically-dead fraction on failure words, and 0.0% on `pidafikawo` +- **less `ReplayOnto` cloning** — bounded by the allocation work's consistent finding that + allocation reduction converts to wall clock at well under 1:1 on this codebase + +Expected contribution: 10–20% of the residual. If Stage 0's measured split shows the residual is +smaller than the ceiling doc's derived 2.8 s, skip this stage and say so. + +**Explicitly out of scope for this branch:** the static rule-pair commutativity analysis. It is +the highest-value unbuilt work in this area — it is what would let order variants merge at +readout and make the F1 probe's 28.72x synthesis-input dedup reachable — but it is a separate +piece of research with its own soundness burden, and "merge on multiset equality alone" is +already *proven* unsound on two grammars. It gets its own branch. + +--- + +## 8. PR description (lift this) + +> ### Narrow the analysis memo key to the rules that can actually loop +> +> `AnalysisStateKey` carries a per-rule unapplication-count multiset spanning every +> morphological rule in the grammar. Since `MaxApplicationCount` defaults to 1, that component +> is effectively *which subset of rules has been unapplied* — up to 2^k distinct keys for k +> rules, all with the same shape and the same feature structures. Most of those distinctions do +> no work: a rule whose unapplication shrinks the word cannot drive an infinite regress, because +> shape length is already a decreasing measure. +> +> This PR classifies each morphological rule by its length effect under unapplication, keeps only +> the non-shrinking rules in the memo key, and moves the per-rule unapplication limit from the +> search into a post-analysis filter. Termination is preserved by a lexicographic measure: shape +> length strictly decreases along a shrinking edge, and the retained-rule count strictly +> increases along a non-shrinking one, so no cycle can close — including the zero-morpheme +> N -> V -> N cycle, which the `>=` (not `>`) boundary keeps in the key on purpose and which is +> covered by a named fixture. +> +> The classifier is conservative by construction: anything not *provably* shrinking stays in the +> key, which is the status quo and always sound. +> +> **Why it matters beyond the cascade.** `AnalysisScope.TemplateMemo` is keyed by the same +> object, and the affix-template battery runs once per distinct key. Narrowing the key reduces +> battery runs one for one — and the battery was 93% of wall time before it was memoized. That +> is the mechanism by which a change to a 1.4 s component moves a 6.1 s word. +> +> **Measured:** _(Stage 0 / Stage 2 table goes here — R per grammar, wall-clock per heavy word, +> synthesis input counts)_ +> +> **Verification.** Analysis-set equality (canonical morpheme-signature sets, not byte +> equality) against the pre-change memo path on Sena, Indonesian and Amharic, in both flag +> states, plus new unit coverage for the classifier, key equality, the count-limit filter, and +> the zero-morpheme cycle. +> +> **Scope.** This does not make HermitCrab polynomial end to end, and the PR does not claim to. +> Analysis becomes polynomial in input length for template-only grammars — the restricted class +> where the key collapses to (shape, stratum, feature structures) — but the grammar constant is +> a feature-structure lattice, and synthesis remains path-enumerated. The exponential that +> dominates our pathological words lives at the analysis/synthesis interface, not in the state +> graph; see `docs/hermitcrab-packed-forest-research.md` for the measurements and for why +> merging order variants at readout is unsound without a commutativity analysis we have not +> built. + +--- + +## 9. Provenance + +The proposal is an external correspondent's, in an email thread about +Maxwell & Kaplan 1993 (`https://aclanthology.org/J93-4001/`). The termination argument, the +`>=` boundary, the `` pairs, and the static length classification are theirs. +The measurements, the ceiling analysis, and the scoping are ours. `docs/ +hermitcrab-packed-forest-research.md` records both, including where the proposal's model of the +engine and ours differ. diff --git a/docs/hermitcrab-optimization-ledger.md b/docs/hermitcrab-optimization-ledger.md new file mode 100644 index 00000000..0dac23e1 --- /dev/null +++ b/docs/hermitcrab-optimization-ledger.md @@ -0,0 +1,106 @@ +# HermitCrab optimization ledger — tried, closed, do not retry + +**Purpose: stop this work being redone.** One row per optimization attempted, what was expected, +what actually happened, and the number that settled it. If you are about to try something on this +list, read its row first — several of these look irresistible on paper and three of them have +already been independently rediscovered and re-retracted. + +Deliberately in-repo and durable, not evicted into a PR accordion. A closed avenue is only closed +if the next person can find out cheaply that it is closed. + +Sources: `hermitcrab-packed-forest-research.md` (theory + prior branches), +`hermitcrab-forest-memo-plan.md` (key narrowing), `hermitcrab-forest-memo-ceiling.md` +(predictions), `hermitcrab-synthesis-fold-probes.md` (measurements, sections 6–10). + +--- + +## Shipped and working + +| # | Optimization | Expected | Outcome | Number | +| --- | --- | --- | --- | --- | +| 1 | Memoize analysis morphological-rule cascade | Same state reached by many rule orders; cache it | Works, but the cascade was never the cost | 2,555 expansions vs a 2,546-state floor; 1.4 s of a 30.5 s word | +| 2 | Memoize affix-template battery (same key) | Battery was 93% of wall time | **The big win.** Reduced how *often* the battery runs | 38,840 runs -> 2,581; word 30.5 s -> **6.1 s** | +| 3 | Shape sharing at clone (`CloneShareFrozenShape`) | Eager deep-copy of Shape dominates allocation | Works, allocation only | **-4.5 to -7.6% bytes**, no wall change | + +## Closed — measured, does not pay + +| # | Optimization | Expected | Why it failed | Number | +| --- | --- | --- | --- | --- | +| 4 | **Narrow `AnalysisStateKey`** (drop strictly-shrinking rules' un-application counts) | `MaxApplicationCount` defaults to 1, so the count multiset is "which subset of rules was used" — 2^27 states. Dropping 19 of 27 rules gives 2^8 | Shape + syntactic FS + realizational FS *already* discriminate. The count component was implied by its neighbours. A worst-case bound collapsing is not the realised state count collapsing | Realised state collapse **Sena 1.12x** vs a 1.3 gate; 6 of 7 heavy words exactly **1.00**; `atawirambo` 2,556 -> 2,556 | +| 5 | **Fold-step sharing memo** (share deterministic synthesis fold steps by computed value) | Probe measured 3.22x and 8.10x shareable steps; ~50% wall ceiling on two reliable fixtures | A sound key must carry the **ordered remaining trail**. Two candidates then have to agree on their entire future to share a step, which almost never happens | **hits = 0** on the best fixture; **0.96x** (4.5% *slower* — pure key-construction cost); 1.06x on the second, inside 21.9% noise | +| 6 | **Order-insensitive synthesis-input dedupe** | 9,774x duplicate synthesis inputs | HermitCrab's morphological rules are **not order-invariant**: same rule multiset, different order, different output. Merging loses parses | 2 violations Sena, 12 independently Indonesian (e.g. `{meN, -Cont}`). Sound version: **15–40%** | +| 7 | **Trail-position indexing of synthesis rules** | 11,445,538 of 11,445,538 rejections are "this rule is not the pending trail rule" — 100% of the histogram | Each rejection is one array index plus one reference compare. Count is not cost | 11.4M x 29 ns = 0.33 s of 143 s = **0.2%** | +| 8 | Synthesis-side length bound (Gate A) | Reject candidates that cannot reach the surface length | At the comparison point the candidate is still the bare root; its affix trail applies later inside `_synthesisRule.Apply`. Rejected valid parses | Unit suite 64 -> 34 passing. Reverted | +| 9 | Analysis length ceiling (Gate B) | Same idea, analysis side | Sound and correct. No available corpus contains the pathology it prunes | Byte-identical output, **no measurable speedup** (~4–7% slower, within noise) | +| 10 | Lexical reachability gate (Phase 5) | Prune candidates from which no lexicon root is reachable | Both reference corpora have real compounding in their deepest stratum, which disqualifies the gate everywhere | Proven **no-op** on both corpora | +| 11 | Tandem lexical intersection (T2) | Kill doomed branches early via the lexicon | The oracle only sees lexical unreachability. The expensive words fail on checks that run *after* lexical lookup succeeds | Pooled **23.5%** dead steps vs a 30% gate; one failure word at exactly **0.0%** dead | +| 12 | Pool small short-lived collections | Fewer allocations, faster | `HashSet/Dictionary.Clear()` is O(capacity); Gen0 already beats pooling at this size | -15–17% bytes but **+8.6% wall**, +12% on the parallel battery. Reverted | + +## Closed before building — motivation refuted by measurement + +| # | Optimization | Expected | Why it died | +| --- | --- | --- | --- | +| 13 | Surface-length pruning inside the synthesis fold (P2) | "The only lever on Amharic's ~160 ms per synthesis run" | Amharic's forward synthesis is **14 ms**. The 160 ms was arithmetic on an unmeasured denominator — dividing 30 s by 186 synthesis inputs and assuming the time was there | +| 14 | Nogood lattice subsumption (P4) | Nogood hits dominate real hits 434,628 to 25,102; generalise "empty" over the feature lattice | Shape + features already discriminate almost perfectly (see #4), so subsumption buckets hold one member and nothing generalises | +| 15 | Move constraints into the analysis key (Maxwell & Kaplan's category-splitting) | Their biggest measured win: make the chart prune what the constraint solver otherwise would | Everything synthesis rejects on is root- or realization-dependent, and the root is unknown until `LexicalLookup`. Also a Sena analysis state costs ~0.73 ms while a synthesis input costs at most ~0.12 ms — **states are dearer than synthesis inputs** | + +## Open + +| # | Optimization | Status | +| --- | --- | --- | +| 16 | **`ExpandAlternatives` dedupe** | Live. 20.3% of Sena wall time, never instrumented before this work. 926 alternatives -> 3 distinct on one fixture, **100% same-analysis-word** so interceptable pre-expansion. Sound ceiling **3–8% Sena**. Needs a trail-complete key | +| 17 | **Per-node cost in the analysis cascade** | Live and unexplored — see below | +| 18 | Corpus-scope memoization (P3) | Not run. `AnalysisScope` dies per word but the key is word-independent by construction. Amharic's corpus run takes 4.3 hours | +| 19 | Contexted constraints / abstract feature-only replay | Not built. Gated on a die-point histogram that says candidates die on shape-free checks | +| 20 | Generation (`GenerateWords`) | **Unmeasured.** Pure synthesis, no analysis phase, so share ~100% — the fold-step ratios would apply at face value. Every verdict above is scoped to *parsing* | + +--- + +## The lesson that generalises + +Three independent measurements, three boundaries, same collapse: + +| measurement | key used | apparent | sound | +| --- | --- | --- | --- | +| synthesis-input dedupe (#6) | order-insensitive | 9,774x | 15–40% | +| fold-entry census | trail position only | 6,476x | not established | +| fold-step sharing (#5) | trail position only | 3.22x / 8.10x | **hits = 0** | + +**The redundancy in HermitCrab's synthesis is apparent, not real. The trail is what makes each +step distinct, and every measurement showing large shareable work is measuring a key that omits +it.** The thing that makes sharing visible is the thing that makes sharing wrong. + +That closes a *family*: packed parse forests, fold-step sharing, and synthesis-input dedupe all +require distinct derivations to converge on a genuinely identical state. In this engine they do +not converge. It is the same fact as the rules being non-order-invariant, seen from the other side. + +## Where the time actually is + +Measured with eight exclusive buckets summing to wall (`unaccounted` 0.1% on Amharic, 1.5% on Sena): + +| grammar | breakdown | +| --- | --- | +| **Sena** heavy words | analysis template battery **51.4%** (still, after the 5x memo), `ExpandAlternatives` **20.3%**, analysis cascade 18.4%, **all synthesis ~5%** | +| **Amharic** | analysis cascade **~95%** of analysis, which is 99.5% of wall; **all synthesis 0.3%** | + +**Every optimization in this ledger that worked reduced how many nodes are visited. None touched +what a node costs.** Amharic spends **~170 ms per analysis state** in a cascade already at its +state-count floor. That is the unexplored axis and the reason #17 is open. + +## Method rules earned here + +1. **Counting is not timing.** Three published findings in this work were retracted, all arithmetic + on an unmeasured denominator: "synthesis is the bottleneck" (candidate counts), "Amharic is + synthesis-bound" (30 s / 186 inputs), "fixtures only look synthesis-heavy because they are small" + (divided by `synForward`, which excludes the buckets the work runs in). +2. **A ratio measured with an incomplete key is not a ratio.** See the table above, three times. +3. **Warm-up dominates small fixtures.** A first A/B here showed 1.53x purely from JIT; the off-arm + alone varied 42.7% between its own first and second sample. Discard warm-up, take min of N, + interleave arms, and print the off-arm spread as a noise floor beside every speedup. +4. **Sub-50 ms fixtures cannot carry a timing claim.** One 10 ms fixture's share moved 1.3% -> 14.5% + between two runs of identical code. +5. **Three grammars minimum, reported unpooled.** Maxwell & Kaplan measured a 100x swing between two + variants of one grammar. A pooled average across fixtures of different size has had to be + retracted twice in this project. +6. **Search completeness is never traded for speed.** HermitCrab is the permanent fallback engine + behind the FST work. diff --git a/docs/hermitcrab-packed-forest-research.md b/docs/hermitcrab-packed-forest-research.md new file mode 100644 index 00000000..d89b3ea1 --- /dev/null +++ b/docs/hermitcrab-packed-forest-research.md @@ -0,0 +1,352 @@ +# Packed parse forests and state memoization in HermitCrab: a research record + +Status: research record, long-term storage. Not a plan — the plan is +`hermitcrab-forest-memo-plan.md`; the ambition analysis is +`hermitcrab-forest-memo-ceiling.md`. + +Written 2026-08-26 on `feature/forest-memo` (off `feature/memoization`). This file exists so +the next person to pick this up does not have to re-derive the theory, re-read the paper, or +re-run the probes. Every number below is traceable to a named branch, commit, or paper page. + +--- + +## 1. What was proposed + +An external correspondent proposed making HermitCrab's analysis phase polynomial for +template-only grammars by changing what goes into `AnalysisStateKey`: + +1. Before a memo lookup, **drop from the key any rule whose unapplication strictly shrinks the + word.** Keep rules whose unapplication leaves the word the same length or longer. +2. Always record a `` back-edge on each key, so the key set plus the edges + form a **packed parse forest** rather than a set of independent memo entries. +3. Recover full derivations at readout by walking the back-edges, and **move the per-rule + unapplication-count limit from the search to the readout**, filtering there. + +The termination argument offered: along a shrinking edge the shape strictly shortens; along a +non-shrinking edge the retained rule count strictly increases. A cycle needs one or the other, +so no cycle can close. The `>=` boundary (rather than `>`) is what keeps a zero-morpheme +N -> V -> N loop out of the forest. + +The claimed bound: for a template-only grammar the analysis states are substrings of the input +(O(n^2) of them), each derivable O(n) ways, giving O(n^3) — O(n^2) with bounded affix length +and no compounding — times a grammar constant "something like the number of distinct required +features in the rules." + +### 1.1 What is right about it + +**The termination argument is valid.** It is a well-founded ordering on the pair +(shape length, retained-rule count), lexicographic with the second component increasing. It is +also *sharper* than what the current key does: it identifies precisely which counts are +load-bearing for termination and licenses discarding the rest. + +**It names a real exponential.** `AffixProcessRule.MaxApplicationCount` and +`CompoundingRule.MaxApplicationCount` both default to `1` +(`MorphologicalRules/AffixProcessRule.cs:28`, `MorphologicalRules/CompoundingRule.cs:19`). +With a limit of 1, the `_ruleCounts` component of `AnalysisStateKey` degenerates to *which +subset of the grammar's rules has been unapplied so far* — 2^k distinct values for k rules, in +the worst case, all sharing one shape and one pair of feature structures. That is a genuine +worst-case exponential sitting in the key, and removing the shrinking rules from it is the +right way to attack it. + +**The static classification is feasible and half-built.** `GrammarAnalyzer` on +`parse-optimization-archive` already reasons about rule length effects +(`ComputeMaxAnalysisLength` bounds net insertion per affix rule and net restoration per +phonological deletion subrule) and already knows the direction trap — un-applying a deletion +rule *inserts*. `IsEdgeStripperQualified` already walks `AffixProcessAllomorph.Rhs` action by +action classifying `CopyFromInput` / `ModifyFromInput` / `InsertSegments` / +`InsertSimpleContext`. A length-effect classifier is a small addition to code that exists. + +### 1.2 What it gets wrong about HermitCrab + +**"The words in the AnalysisStateKeys are substrings of the input" is false in general and +shaky even for templates.** `AnalysisStateKey` is not a string. It is +`(Shape, Stratum, SyntacticFeatureStruct, RealizationalFeatureStruct, NonHeadCount, +ruleCounts)`. Unapplying an affix template rule writes required head features back into the +word's syntactic feature structure; realizational rules accumulate into +`RealizationalFeatureStruct`. So the state space is (substrings) x (a feature-structure +lattice), and that lattice is exponential in the number of features, not linear in the number +of categories. The proposal's "grammar constant" is doing very heavy lifting. This is not a new +observation — it is the standard result for constraint-based formalisms (Barton, Berwick and +Ristad 1987; see section 2). + +**"Read out only happens when a stem matches the lexicon, and the time to perform read out +should be linear in the number of analyses produced" mis-models the engine.** HermitCrab's +readout is not a readout. `Morpher.Synthesize` (`Morpher.cs:299` / `:310`) takes every analysis +candidate, does `LexicalLookup`, expands `ExpandAlternatives`, re-runs the whole synthesis rule +cascade forward, and only then checks `IsMatch(word, validWord)` against the original surface +string. Candidates that survive analysis and lexical lookup and still fail are the normal case, +not the exceptional one. Measured: `cinacemerwa` (Sena) produces **218,847 synthesis inputs and +returns 0 parses**. There is nothing to enumerate; the entire cost is proving that. + +**Moving the count filter to readout removes pruning from the cheap phase and adds work to the +expensive one.** As a set-equality claim it is sound: an analysis is valid iff every rule's +count is within its limit along that path, so filtering during search and filtering at +enumeration yield the same set. But the search-time filter currently stops those paths before +they reach synthesis, and synthesis is where the money is. This has to be measured, not assumed +in either direction — see the ceiling doc. + +--- + +## 2. The theory: Maxwell & Kaplan 1993 + +The correspondent's link, and the right frame for the whole discussion: + +> John T. Maxwell III and Ronald M. Kaplan. "The Interface between Phrasal and Functional +> Constraints." *Computational Linguistics* 19(4):571–590, 1993. +> https://aclanthology.org/J93-4001/ + +The paper is about hybrid systems that split into a **context-free phrasal component** +(polynomial, packable into a chart or forest) and a **functional constraint component** +(unification/equality, exponential in the size of the constraint system). Its subject is the +*interface* between them. The mapping to HermitCrab is close enough to be useful and different +enough to matter: + +| Maxwell & Kaplan | HermitCrab | +| --- | --- | +| context-free phrasal constraints | analysis cascade — unapplying morphological rules | +| chart edge; equivalence of edges | `AnalysisStateKey` | +| parse forest (nested free-choice form) | the proposed `` back-edge forest | +| functional constraints (unification) | feature unification, allomorph co-occurrence, `IsWordValid` | +| checking a solution | forward synthesis + `IsMatch` against the surface string | + +### 2.1 The findings that transfer + +**(a) The exponential lives at the interface, not in either component.** p.572: + +> "even though a context-free parser can very quickly determine that those trees exist, if the +> grammar is exponentially ambiguous then the net effect is to produce an exponential number of +> potentially exponential functional constraint problems. ... This exponential does not come +> from either of the components independently; rather, it lies in the interface between them." + +That is exactly the shape of our measurements. The analysis cascade is cheap and already at its +state floor (section 3.1). The cost is the fan-out from analysis states into synthesis problems. + +**(b) Edge equivalence must account for everything downstream reads.** p.574: + +> "the notion of equivalence must also be augmented to take account of the constraints: two +> edges are equivalent now if, in addition to satisfying the conditions specified above, they +> have the same constraints (or perhaps only logically equivalent ones)." + +and the consequence: + +> "there can be a different set of constraints for every way in which a particular category can +> be realized over a given substring. ... the algorithm becomes exponential in the worst case." + +This is the key-completeness problem, stated in 1993. `AnalysisStateKey`'s doc comment already +carries a hand-audited key-completeness argument against every `Analysis*.cs` rule; the F1 probe +(section 3.2) found the same failure mode empirically at a *different* boundary. Any key +narrowing must re-run that audit. + +**(c) Packing and pruning are in tension, and pruning is not always right.** Section 2.4, +"Still Exponential", p.576: + +> "Although pruning can eliminate an exponential number of trees, this strategy is still +> exponential in sentence length in the worst case when the grammar is exponentially ambiguous +> with few constituents that are actually pruned." + +and from the abstract: + +> "A surprising outcome is that under certain circumstances an algorithm that does no pruning in +> the interface may perform significantly better than one that does." + +Their measurements (Tables 2 and 3, pp.586–587; scaled so interleaved pruning on the base +grammar = 100) bear this out and are worth internalising: + +| Grammar | Strategy | Benchmark unifier | Contexted unifier | +| --- | --- | --- | --- | +| Base | interleaved pruning | 100 | 42 | +| Base | factored extraction (no interface pruning) | >1000 | >1000 | +| Modified | interleaved pruning | 38 | 26 | +| Modified | factored extraction (no interface pruning) | 21 | **7** | + +The same no-pruning strategy is the worst option on one grammar and the best on the other — a +100x-plus swing between two variants of the *same* grammar. That is a standing warning about our +own probe methodology. + +**(d) The way to make the forest pay is to move discriminating features into the phrasal +component.** The "modified" grammar above is the base grammar with categories split so that +features which would otherwise be checked functionally are checked by the chart instead (V into +V_AUX / V_OBL / V_TRANS / V_OTHER, N into N_OBL+ / N_OBL-, and so on; p.585). Every strategy +improves on the modified grammar, and factored extraction improves by 50x. Their citation for +this is Nagata 1992's finding that a medium-grain phrase structure grammar beats both a +coarse-grain and a fine-grain one. + +**For HermitCrab this is the most actionable idea in the paper, and it points the opposite way +from the proposal.** Putting `SyntacticFeatureStruct` into `AnalysisStateKey` already *is* the +medium-grain move. The right instinct is not "shrink the key"; it is "keep in the key exactly +what discriminates, drop exactly what does not." The *order* of shrinking-rule unapplications +does not discriminate. Whether the *set* of shrinking rules unapplied discriminates is an +empirical question — see the plan's Stage 0. + +### 2.2 The finding that does not transfer — and this is the important one + +Section 3.4, "Order Invariance", p.578: + +> "Phrasal constraint systems and functional constraint systems commonly used for linguistic +> description have the property that they can be processed in any order without changing the +> final result." + +**HermitCrab's morphological rules do not have this property, and we have measured it on two +unrelated grammars.** The F1 forest probe on `parse-forest-tandem` found candidate pairs with +*the same pending-rule multiset, a different application order, and different synthesis output*: +2 such pairs on Sena, and independently 12 on Indonesian (e.g. `{meN, -Cont}`). Two grammars +with no shared ancestry, same failure mode — a real property of the formalism, not a probe +artifact. + +Order invariance is the assumption under which a packed forest can be *read out* packed. Without +it, every distinct order is a distinct solution, and the forest can be built compactly but must +be enumerated in full. This single fact separates the 28x figure from the 15–40% figure in +section 3.2, and it is why the honest description of a packed forest in HermitCrab today is "a +representation and memory optimization" rather than "a polynomial parser." + +Note carefully what this does *not* invalidate. The shipped `AnalysisStateKey` uses an +order-independent multiset and is sound, because no *analysis-side* rule reads trail order. The +non-commutativity shows up on the *synthesis* side. Analysis-side order-independence and +synthesis-side order-dependence coexist, and conflating them is the easiest mistake to make in +this area. + +### 2.3 Other literature, already surveyed + +`docs/hermitcrab-parse-algorithm-analysis.md` (complexity-cap branch) carries the verified +survey: Sheil 1976 on the polynomial bound depending on edge equivalence being independent of +daughter substructure; Barton, Berwick & Ristad 1987 on feature systems as the source of +intractability; Karttunen/Beesley "overanalysis" and Koskenniemi tandem lookup for the +lexical-intersection idea. HermitCrab itself is Michael Maxwell's design — a different Maxwell +from the author of J93-4001, worth stating once so nobody assumes a lineage that is not there. + +--- + +## 3. What we have already measured + +All numbers below are from this repository. Sources are named so they can be re-run. + +### 3.1 The analysis cascade is already at its state floor + +From the Phase 3b instrumentation on `parse-optimization`, measured on Sena `atawirambo`: + +- fair sequential unmemoized baseline: **30.5 s** +- morphological-rule cascade after the Phase 2/3 memo: **2,555 node expansions against a + 2,546-state floor** — 0.4% off optimal — and **1.4 s of the 30.5 s** +- affix-template battery: **93% of wall time**, run **38,840x** against **~2,581 distinct keys** +- after memoizing the template battery too: 30.5 s -> **6.1 s** (`cinacemerwa` 102.7 -> 26.9 s) + +The subsystem the proposal makes polynomial is the 1.4 s one, and it is already within 0.4% of +its own state-count floor. Even making it free saves about 5% of the unmemoized word. + +**But there is a second-order effect the proposal does not mention, and it is the strongest +argument in its favour.** `AnalysisScope.TemplateMemo` is keyed by the *same* +`AnalysisStateKey`. The template battery runs once per distinct key. Narrowing the key therefore +reduces template-battery runs **one for one** — and that is the subsystem that was 93% of the +cost. If key narrowing halves the state count, it halves the battery runs. This is the mechanism +by which a change to a 1.4 s component can move a 6.1 s word. + +### 3.2 The forest's dedup ceiling, and why it is not reachable today + +From the F1/F2 probes on `parse-forest-tandem` (commits 59d1e730, 5215fc01, f09714ba..7409cf40): + +- dedup of synthesis inputs on an **order-insensitive** key: **28.72x** aggregate on the Sena + heavy words (5 of 7 individually clear 3x; `manyeredzero` 2.89x and `pidafikawo` 3.81x are + marginal). Indonesian: **1.41x** — the win is Sena-shaped, not universal. +- that key is **unsound**: 2 residual violations on Sena, 12 on Indonesian, all genuine rule + non-commutativity (section 2.2). +- dedup on the **fully order-sound** key (F2 as shipped): **~15–40%** call reduction on Sena + heavies. + +An earlier probe iteration reported 9,774x. That number was inflated by a key that conflated +genuinely different rule sets. It is recorded here only so nobody rediscovers and believes it. + +### 3.3 The lexical-reachability oracle does not rescue readout + +The T1 probe (same branch, finalized commit aaeb2b35) asked how many cascade steps are provably +dead because no lexicon root can be reached from that node. Failure words — the expensive ones — +came in at `cinagumanika` 32.6%, `cinacemerwa` 24.4%, `manyeredzero` 18.5%, `pidafikawo` 0.0%; +pooled step-weighted **23.5%**, under the 30% build gate. `pidafikawo` at exactly 0.0% is the +diagnostic case: a root substring exists at every node visited and the word still fails, because +it fails on checks that happen *after* lexical lookup succeeds — environments, co-occurrence, +disjunctive allomorphs, syntactic features, surface match. A forest that only prunes +lexically-dead states does not prune those. + +Consequence for the proposal: routing readout only through lexicon-live states — the obvious way +to exploit the forest — recovers at most the dead fraction, roughly a quarter on the words that +hurt. + +### 3.4 A hidden coupling in the shipped memo, found while designing the narrowing + +`Word.ReplayOnto` splices two things when it grafts a stored subtree onto a new arrival: the +ordered rule trail `_mruleApps` and the non-head list `_nonHeadApps`. It does **not** splice +`_mrulesUnapplied`, the per-rule un-application count dictionary — the replayed word simply +inherits the stored result's copy. + +That is correct today, but only by accident of the key. Because `AnalysisStateKey` includes the +full count multiset, a memo hit guarantees the arriving word and the stored entry's own arrival +word had *identical* counts, so the stored result's counts are already the right ones. The key's +count component is silently doing double duty: it is not only a state distinction, it is what +makes `ReplayOnto`'s omission safe. + +**Any narrowing of the count component breaks that invariant** — arrival and stored-arrival can +then differ on exactly the dropped rules, and the replayed word inherits counts that were never +its own. Anything reading `Word.UnappliedRuleCounts` after a replay (such as a post-analysis +limit filter, which is precisely what the proposal calls for) would read wrong numbers. + +The fix is small and worth doing regardless, because it removes the hidden coupling: store the +arrival's counts on `MemoEntry` and have `ReplayOnto` compute +`clone.counts = stored.counts − storedArrival.counts + query.counts`. With the full key the +delta is zero and the change is a no-op, which is also how it should be tested. + +Recorded here because it is a property of the shipped memoization, not of any proposal, and the +next person to touch the key needs to know about it. + +### 3.5 Things already closed, so they are not re-proposed + +- **Tandem lexical intersection (T2): not built.** Gate not met; mechanism understood (3.3). +- **Gate A (synthesis-side length bound): reverted.** At the point of comparison the candidate + is still the bare root; its affix trail applies later inside `_synthesisRule.Apply`. Any future + length reasoning on the synthesis side hits this same wall. +- **Phase 5 lexical gating: a proven no-op** on both reference corpora, because both have real + compounding in their deepest stratum. +- **Pooling of small short-lived collections: reverted, net loss on every axis.** `Clear()` is + O(capacity); Gen0 beats pooling here. + +--- + +## 4. Where the open questions actually are + +Ranked by how much they gate the outcome. + +1. **Static rule-pair commutativity.** If pairs of morphological rules can be shown + order-independent — by static analysis of their allomorph `Rhs` actions and feature effects, + or by a verify-once-per-equivalence-class dynamic check — then order variants can be merged at + readout and the 28x becomes reachable. Without it, everything else is bounded by section + 3.2's 15–40%. This is the highest-value unbuilt work in this area. +2. **Does the shrinking-rule *set* discriminate?** Two paths that strip different affix sets and + land on the same shape and feature structures exist only where the grammar has homophonous + affixes. Sena has many. Directly measurable before any code is written — Stage 0 of the plan. +3. **Does readout-time count filtering inflate the synthesis input set?** Set-equivalent, but the + work moves from the cheap phase to the expensive one. Measure synthesis input counts, not just + wall time. +4. **Can any part of synthesis run on a packed representation?** Maxwell & Kaplan's + contexted-constraint question, transposed. Feature unification and allomorph co-occurrence + plausibly can. The phonological rewrite cascade almost certainly cannot — it is a sequential + transduction, not a constraint system. Answering "which half of synthesis is a constraint + system" would decide whether a genuinely polynomial end-to-end parser is available at all. + +--- + +## 5. Standing methodological rules for this area + +Earned the hard way on the branches cited above. + +- **Counting redundant expansions is not counting cost.** 98% measured redundancy yielded ~32% + wall clock, because guard clauses reject cheaply before FST matching. Always take the fair + same-mode baseline and split wall time by subsystem before believing a redundancy ratio. +- **A dedup ratio measured against an unsound key is not a dedup ratio.** 9,774x -> 28.72x -> + 15–40% is one measurement getting progressively honest. +- **Three grammars, always.** Maxwell & Kaplan's own 100x-plus swing between two variants of the + same grammar is the argument. Sena, Indonesian and Amharic behave differently enough that any + one of them alone will mislead. +- **The acceptance gate is analysis-set equality, not byte equality.** A memo-replayed `Word` is + not field-for-field identical to a freshly computed one. Compare canonical + morpheme-signature sets. +- **Search completeness must never be reduced.** Standing owner constraint: HermitCrab is the + permanent fallback engine behind the FST work, so a faster parser that loses parses is not a + faster parser. diff --git a/docs/hermitcrab-synthesis-fold-probes.md b/docs/hermitcrab-synthesis-fold-probes.md new file mode 100644 index 00000000..8c193d38 --- /dev/null +++ b/docs/hermitcrab-synthesis-fold-probes.md @@ -0,0 +1,628 @@ +# Synthesis-fold probes: plan + +Branch: `feature/synthesis-fold-probes`, off `integrate-conformance-framework` (`c0ac5c9f`). + +Deliberately **not** off master or `feature/memoization`. The conformance branch carries 33 +committed grammars — 8 typologically distinct languages (fusional-realizational, metathesis, +polysynthetic-stratal, prefixal-discontinuous, suffixing-evidential, suffixing-extension-slot, +suffixing-vowel-harmony, templatic-root-modification) plus 25 edge cases — each with hand-derived +expected outputs and a `Fixture.DiscoverAll` enumerator. Every prior conclusion in this area was +drawn from three grammars, and the last one died because a result that looked general was +Sena-shaped. Breadth is the point of this base. + +Predecessor: `feature/forest-memo`, where the proposal to narrow `AnalysisStateKey` was measured +and stopped. Its three docs (`hermitcrab-packed-forest-research.md`, +`hermitcrab-forest-memo-ceiling.md`, `hermitcrab-forest-memo-plan.md`) hold the evidence base and +are ported here unchanged. Read them first; nothing below re-derives them. + +--- + +## 1. The reframing this plan tests + +Every previous attempt to share synthesis work assumed sharing means **merging by key**, which +requires order-invariance — and HermitCrab's morphological rules provably lack it (2 violations +on Sena, 12 independently on Indonesian). + +But forward synthesis is not order-sensitive search. It is a **deterministic fold driven by the +trail**. Verified in code: + +- `Word.IsMorphologicalRuleApplicable` (`Word.cs:248`) admits **only** + `_mruleApps[_mruleAppIndex]`; `MorphologicalRuleApplied` decrements the index. Synthesis walks + the trail from its end backwards, branching only on allomorph choice. +- The **end** of `_mruleApps` is the last-unapplied rule — the deepest, the one applied **first** + in synthesis. `ReplayOnto`'s `mruleTrailPrefixLength` splits the trail at exactly that + boundary. So **the analysis memo has already computed which candidates share a synthesis-first + segment**, and that segment is currently re-folded from scratch for every path, every root, + every alternative. + +Two ways to share a deterministic fold, neither of which assumes commutativity: + +1. **By guaranteed-identical subsequence** — the shared suffix above, already identified for free. +2. **By computed value** — if the extension step is a function of the partial's full value plus + the next trail rule, then two permuted segments that *in fact* produced the same value merge + automatically, and the rare genuine non-commutative pairs produce different values and stay + separate. + +Order-invariance stops being an assumption and becomes something detected. That is the whole +idea, and P1c is the measurement that decides whether it is worth anything. + +### Two traps already found in the code + +- **`Word.ValueEquals` (`Word.cs:600`) is not a valid synthesis fingerprint.** It compares shape, + realizational FS, non-heads, stratum, root allomorph, trail, index, and the final-rule flag — + and **omits `_syntacticFS`, MPR features, and disjunctive allomorph indices**, all of which + synthesis reads. A `SynthesisStateKey` needs its own key-completeness audit against every + `Synthesis*.cs`, exactly as `AnalysisStateKey` has one. +- **Realizational rules are trail-exempt.** `SynthesisRealizationalAffixProcessRule` has no + `IsMorphologicalRuleApplicable` gate (contrast `SynthesisAffixProcessRule.cs:43`); it gates on + `RealizationalFeatureStruct.Subsumes` plus `IsBlocked`. They branch *inside* a shared segment, + so any stored partial must be a set, like `MemoEntry.Results`, not a value. + +### One clean negative, recorded so it is not re-attempted + +Maxwell & Kaplan's biggest measured win came from modifying the grammar so the chart prunes what +the constraint solver otherwise would. **That cannot transfer here.** Everything synthesis +rejects on is root- or realization-dependent, and the root is unknown until `LexicalLookup`. On +top of that, a Sena analysis state costs ~0.73 ms (template battery) while a synthesis input +costs at most ~0.12 ms — **states are dearer than synthesis inputs**, and that conclusion gets +stronger, not weaker, if synthesis turns out to be less than all of the runtime. The chart's +analogue of category-splitting here is the interface, not the key. + +--- + +## 2. Standing rules + +1. **Search completeness is never reduced.** HermitCrab is the permanent fallback engine behind + the FST work. +2. **Acceptance is analysis-set equality**, never byte or object equality. Canonical + morpheme-signature sets, sorted. +3. **Breadth before depth.** Every number is reported across all 33 conformance fixtures *and* + Sena/Indonesian/Amharic. A result that holds only on Sena is not a result — that is exactly + how the key-narrowing work went wrong. +4. **Gates are written before the run**, and a missed gate is reported as a finding, not + renegotiated. +5. **No pooled averages across fixtures of wildly different size.** Report per-fixture, plus a + distribution. The T1 tandem probe had to retract a pooled average; the forest-memo Amharic + number nearly repeated it. +6. **Real grammars (Sena/Indonesian/Amharic) are never committed.** Conformance fixtures are + synthetic and committed; corpus grammars stay in `.git/info/exclude`. + +--- + +## 3. P1 — the synthesis instrumentation triple + +One harness, three numbers. All three instrument the same loop in `Morpher.Synthesize`, so they +are gathered in a single pass rather than three. + +### P1a — the wall-time split + +Per word: time in the morphological-rule cascade, the affix-template battery, `LexicalLookup`, +and forward synthesis (`_synthesisRule.Apply` + `IsWordValid` + `IsMatch`). + +**This is a gap in the predecessor's own work.** The forest-memo Stage-0 plan listed this row as +required and never delivered it, and every ceiling in this plan divides by it. It is cheap and it +comes first. + +No gate — it is a precondition for interpreting P1b and P1c. + +### P1b — the die-point histogram + +`cinacemerwa` sends 218,847 candidates into forward synthesis and returns 0 parses. We have never +asked *why* they die. For each rejected candidate, record which check killed it: + +| die point | where | +| --- | --- | +| lexical lookup miss | `Morpher.LexicalLookup` yields nothing | +| synthesis-side application count | `SynthesisAffixProcessRule.cs:46` | +| morphological rule not applicable / pattern match failure | `SynthesisAffixProcessRule` | +| allomorph environment | allomorph `Environments` check | +| realizational subsumption / blocking | `SynthesisRealizationalAffixProcessRule` | +| feature unification | `RequiredSyntacticFeatureStruct.Unify` chain | +| MPR features | required/excluded MPR check | +| `IsWordValid` | co-occurrence, obligatory features | +| final surface mismatch | `Morpher.IsMatch` | + +**Gate:** if a single die point accounts for **≥40%** of rejections on the Sena heavy words *and* +is decidable from information available before the synthesis cascade runs, build that prefilter. +Report the histogram for all 33 fixtures regardless — the shape of the distribution across +typologies is itself the finding. + +### P1c — the fold-step fingerprint ratio (the headline) + +Count total synthesis rule applications against distinct `(fingerprint, applied rule)` pairs, +where the fingerprint covers everything a synthesis step reads: shape+annotations, syntactic FS, +realizational FS, MPR set, root allomorph, disjunctive allomorph indices, application counts, +`IsPartial`, `IsLastAppliedRuleFinal`, stratum, and pending-trail position. **Do not use +`Word.ValueEquals`** — see the trap above. + +Also assert a **determinism check**: equal fingerprint plus equal applied rule must never yield +different outcomes. A violation means the fingerprint is incomplete and is the single most +important thing this probe can find. + +**Gate:** ratio **≥5x** on `cinacemerwa` and `kukucitirani` → build suffix-anchored synthesis +sharing. **<2x** → the idea is dead, fact 5's pessimism was right, and we say so. +Between 2x and 5x → report and decide with the P1a split in hand. + +--- + +## 4. Later probes, in order, each gated on the last + +- **P2 — incremental surface-length pruning.** Gate A resurrected at the layer where it is sound: + it died because at its comparison point the candidate was still the bare root with the trail + unapplied; *inside* the fold the partial is real and the pending trail is known. + `RuleLengthClassifier` (built on the predecessor branch, currently orphaned) supplies the + min-insertion side. The only proposal that can touch Amharic's ~160 ms-per-run problem. + Gated on P1b: build only if the histogram says candidates die somewhere a length window can see. +- **P3 — corpus-scope memoization.** `AnalysisScope` dies with each word, but the key is + word-independent by construction. Least clever, lowest risk, aimed at the number users feel — + the Amharic corpus run took 4.3 hours. Independent of P1; can run any time. +- **P4 — nogood lattice subsumption.** Lowest priority: if shape+FS discriminate almost perfectly + (which the forest-memo measurement says they do), the subsumption buckets have one member and + this never fires. Probe is a counting exercise; gate ≥30% subsumable. + +--- + +## 5. Execution + +One probe at a time. Each probe is implemented by a subagent against a hardened brief, then its +claims are verified independently before anything is believed. New branch per probe where the +work is separable. A Fable review after the first probe lands, then continue down the list. + +Results land in section 6 of this file as they arrive. + +--- + +## 6. Results + +### 6.1 P1 — conformance breadth (33 fixtures) + +Full HermitCrab suite on this branch with the probe present: **582 passed, 1 skipped, 0 failed**, +including every conformance fixture gate. The instrumentation is behaviour-neutral: all edits are +insertions before existing `return`/`continue` statements, gated on a single +`volatile bool SynthesisProbe.Enabled` that is false in normal operation. + +**Determinism violations across all 33 fixtures: 0.** Equal fingerprint plus equal applied rule +never produced a different outcome multiset, across 8 typologies and 25 edge cases. This is the +check that would have exposed an incomplete fingerprint, and it is clean — which is what licenses +reading the P1c ratios below as real sharing rather than as collisions. + +**P1c fold-step sharing ratio, by fixture** (never pooled — sizes differ by three orders of +magnitude): + +| fixture | applications | distinct | ratio | +| --- | --- | --- | --- | +| languages/suffixing-evidential-adjacency-chain | 640 | 79 | **8.10x** | +| edge-cases/strrep-identity | 67 | 17 | 3.94x | +| edge-cases/deep-optional-affix-nesting | 5,556 | 1,727 | **3.22x** | +| edge-cases/diacritic-segments | 48 | 16 | 3.00x | +| edge-cases/disjunctive-recheck | 12 | 4 | 3.00x | +| languages/suffixing-vowel-harmony | 45 | 16 | 2.81x | +| languages/suffixing-extension-slot-ordering | 88 | 41 | 2.15x | +| languages/templatic-root-modification | 27 | 14 | 1.93x | +| edge-cases/morphotactic-attribute-breadth | 131 | 83 | 1.58x | +| languages/fusional-realizational-morphology | 59 | 40 | 1.48x | +| languages/metathesis-phase-isolation | 10 | 9 | 1.11x | +| languages/polysynthetic-stratal-derivation-chain | 5 | 5 | 1.00x | +| edge-cases/mpr-overwrite-order-dependence | 19 | 19 | 1.00x | + +**Fold sharing is real and strongly typology-dependent.** Suffixing/agglutinative chains share +heavily; metathesis and MPR-order-dependent grammars share nothing. That split is a sanity check +in itself: the fixture literally built to be order-dependent +(`mpr-overwrite-order-dependence`) reports exactly 1.00x, and the metathesis fixture 1.11x, while +a suffix chain reports 8.10x. The measurement discriminates in the direction the mechanism +predicts. + +This is precisely the information the predecessor branch lacked. Key narrowing looked general and +was Sena-shaped; fold sharing is *not* general either, but here we know the shape of the +dependence before building anything. + +**P1b die-point histogram** — consistent across unrelated typologies: +`RuleNotApplicableOrPatternMismatch` 72–73%, `LexicalLookupMiss` 22–25%, everything else in the +single digits. + +Two cautions on reading it: +- These are rejection **events**, not distinct candidates — one candidate branches into many + internal attempts, each able to die at a different check. Documented in `SynthesisProbe`. It is + **not** the same denominator as the historical 218,847 figure, which counts candidates + *entering* synthesis (one per `ExpandAlternatives` output). The two numbers must not be + compared. +- `RuleNotApplicableOrPatternMismatch` clears the 40% gate on count, but each such rejection is an + O(1) trail-position check (`IsMorphologicalRuleApplicable` is a list index plus a reference + compare). A count histogram overstates its cost share. **Cost-weighting is required before this + becomes a build decision** — see 6.3. + +**P1a wall-time split** is unreliable on these fixtures: most words run in well under 2 ms, where +`Stopwatch` overhead and JIT warm-up swamp the signal, and the four buckets frequently sum to well +under half of wall time. The one large-enough fixture is informative: +`deep-optional-affix-nesting` at 2,393 ms with **battery = 67.4%**, forward synthesis 6.2% — +matching the historical Sena finding that the affix-template battery dominates. Treat the split as +meaningful only on the real corpora. + +### 6.2 P1 — Sena, and a correction to how these numbers must be read + +| word | wall | successful apps | new distinct | +| --- | --- | --- | --- | +| `atawirambo` | 19,987 ms | 268 | 32 | +| `kukucitirani` | 89,679 ms | 39,270 | 112 | +| `cinacemerwa` | 61,672 ms | 2,149 | 13 | +| **total** | 171,338 ms | **41,687** | **157** | + +**P1c = 265.5x. Determinism violations: 0.** The gate was ≥5x. + +And on Sena that does not matter, because of the split: + +`lookup 5.68 ms (0.0%) · synthesis cascade 2,629 ms (1.5%) · synthesis battery 5,677 ms (3.3%) · +forward synthesis 328 ms (0.2%)` — **four buckets totalling 5.0% of wall time.** + +Two separate findings are tangled here and must be kept apart. + +**(i) A measurement defect.** All four timers landed on the *synthesis* side — +`AddCascadeTicks`/`AddTemplateBatteryTicks` are called from `SynthesisStratumRule.cs:107`/`:136`, +and `AnalysisStratumRule`/`MemoizedCombinationRuleCascade` were never instrumented. That is an +ambiguity in the brief: both a "morphological-rule cascade" and an "affix-template battery" exist +on each side. Being fixed; the `unaccounted` column is the deliverable. + +**(ii) A real result about Sena.** Even granting the defect, forward synthesis plus lexical lookup +plus the synthesis-side cascade and battery are **5% of Sena heavy-word time**. The 218,847 +synthesis inputs on `cinacemerwa` are real but cost ~0.5 µs each. Counting candidates told us +where the *volume* was and never where the *time* was. This is exactly the row the predecessor +plan required and the predecessor branch skipped, and skipping it cost two rounds of analysis +built on a wrong denominator. + +Corroborating: **11,445,538 rejection events, 100.0% `RuleNotApplicableOrPatternMismatch`**, +against 41,687 successful applications — 274 wasted rule attempts per real one, at ~29 ns each. +Real waste, and the "~40x free" trail-position observation from +`docs/hermitcrab-parse-algorithm-analysis.md` reconfirmed at scale — but at 29 ns it is not 95% of +anything. + +#### The correction to the metric + +Sena being analysis-bound is a fact about Sena, not a verdict on the technique. HermitCrab runs on +a very large number of languages; a technique inert on two grammars and worth 5x on a third is a +useful technique. **The quantity that decides value, per grammar, is:** + +> **value = P1c sharing ratio × forward-synthesis share of wall time** + +We have the first across all 33 fixtures (1.00x–8.10x). We have the second only for Sena, where it +is ~0.2%. **No conclusion about any other grammar is licensed until the second factor is measured +per grammar.** Neither factor alone decides anything: high sharing in a phase that costs nothing +is worthless, and an expensive synthesis phase with no sharing is unimprovable by this route. + +#### Why Amharic is the priority + +From the predecessor branch: `ሄዶ` has **212 analysis states and 186 synthesis inputs, and takes +30 seconds** — roughly **160 ms per synthesis run**, against Sena's ~0.5 µs per synthesis input. +Five orders of magnitude apart per unit of synthesis. If Amharic's wall time sits in forward +synthesis, it is the grammar where fold sharing pays and Sena is the outlier rather than the rule. +Templatic/Semitic morphology is not a niche. This is the measurement that matters most next. + +### 6.3 P1a fixed — where the time actually goes + +Six exclusive slices now sum to wall. **Amharic `unaccounted` = 0.1%**, so the split is +trustworthy. + +Amharic, 28 words: `anTotal` **99.5%** · lookup 0.1% · synCascade 0.1% · synBattery 0.1% · +synForward **0.1%** · unaccounted 0.1%. + +| word | wall | anCascade | anBattery | anPhono | synForward | +| --- | --- | --- | --- | --- | --- | +| `ሄዳችሁ` | 38,298 ms | **34,635** | 3,505 | 8.5 | 41 | +| `ሄዶ` | 37,953 ms | **36,154** | 1,682 | 0.8 | **14** | +| `ሁለተኛ` | 16,486 ms | **14,122** | 2,249 | 44 | 18 | + +**This refutes a claim made earlier in this document.** Section 6.2 argued `ሄዶ` was ~160 ms per +synthesis run and therefore the likely synthesis-bound grammar. Its forward synthesis is **14 ms**. +The 160 ms came from dividing 30 s by 186 synthesis inputs — arithmetic on an unmeasured +denominator, the same error that produced the 218,847 framing. Amharic's 36 seconds are in the +**analysis morphological-rule cascade**, for a word with 212 distinct states: ~170 ms per state, in +a cascade already memoized and already at its state floor. Not state count, not the template +battery (4%), not phonology (0.002%), not synthesis (0.04%). What is expensive is what happens +*inside* the cascade per node — pattern matching across the rule set. + +### 6.4 The answer to "does any grammar benefit?" — CORRECTED + +> **The ceiling table first published in this section was arithmetically wrong and has been +> removed.** It divided by the wrong share. Found by adversarial review, confirmed in code. + +**The error.** `synForward` is explicitly *net* of the cascade and battery buckets — +`Morpher.cs:424` records `forwardTotal - cascadeDelta - batteryDelta`. But every application P1c +counts is recorded inside `SynthesisAffixProcessRule.Apply` / +`SynthesisRealizationalAffixProcessRule.Apply`, which run **inside** the `synCascade` and +`synBattery` brackets (template slot rules compile to those same classes via `RuleBatch`, +`SynthesisAffixTemplateRule.cs:20-24`). So the shareable work lives in +`synCascade + synBattery + synForward`, and the table divided by `synForward` alone — the one +bucket that excludes it. + +**Corrected ceilings** (`share x (1 - 1/ratio)`, share = synCascade + synBattery + synForward): + +| grammar | ratio | corrected share | max possible speedup | +| --- | --- | --- | --- | +| **Sena** | 265x | **5.0%** | **~4.98%** (was reported as 0.2% — understated 25x) | +| **Amharic** | 2.15x | **0.3%** | **~0.16%** | + +**The fixture ceiling column is withdrawn entirely**, for two reasons: the harness only emitted +`forwardShare`, so correcting it needs a re-run; and section 6.1 already declared fixture timings +unreliable at sub-2 ms scale, which makes the former "best anywhere 22.5%" headline +self-contradictory by this document's own standard. Do not quote it. + +**Determinism violations across every run: 0 — but that proves less than previously claimed here.** +`RecordApplications` returns early when `outputs.Count == 0`, so a `(fingerprint, rule)` pair that +succeeds in one occurrence and fails in another is never compared; any omitted state that only +flips match to no-match is invisible to the check. And outcomes are compared with the same +`FingerprintEquals` used to key them, which covers pending-trail *position* but not remaining-trail +*content*. Zero violations licenses **per-step decision determinism**. It does not license "the +sharing is sound": memoized output `Word`s embed trails, so a real build needs delta-storage or +`ReplayOnto`-style re-anchoring. This makes a build harder, not easier. + +#### Reading this honestly + +Two competing explanations for why fixtures show high synthesis share and real grammars show +~0.3-5%: + +1. **Typology.** Some morphological types are genuinely synthesis-heavy. +2. **Grammar size.** Analysis cost scales far worse with rule count and lexicon size than + synthesis does. + +The doc previously asserted (2), on the grounds that Sena is agglutinative and "still 0.2%". **That +premise used the wrong number** — Sena is ~5%. The size argument is now *directionally plausible +but not established*: the large end of the trend is a single fixture whose size comes from an +analysis-side stressor by construction, so size and analysis-pathology are confounded, and most +fixture ratios rest on 12-131 applications. + +There is a concrete mechanism by which a synthesis-bound grammar could exist, and it is in this +document's own trap #2: **realizational rules are trail-exempt**, so their branching is not bounded +by the analysis trail and synthesis work can scale with paradigm size independently of analysis. +The family that would show this — large position-class fusional systems with many realizational +rules per slot and heavy blocking — is represented by none of Sena, Amharic, Indonesian, or any +sub-2 ms fixture. Probe F1 below is designed to settle it. + +#### Verdict on the fold-sharing build — scoped + +**For parsing workloads: do not build it, provisionally.** Amharic's ~0.16% ceiling is robust +under every attack found. **Sena's ~4.98% is provisional on probe N1** — the 20.1% unaccounted +could move it. + +**This verdict does not cover generation.** `Morpher.GenerateWords` (`Morpher.cs:245-254, 805`) is +pure synthesis with no analysis phase — share ~100%, so the P1c ratio applies at face value. +Nothing in this round measured it. + +#### What replaces it + +**The analysis phase is the target on every real grammar — but which part of it differs by +grammar.** Sena re-run with the fixed timers (143,303 ms over the three heavy words): + +| bucket | pooled % of Sena wall | +| --- | --- | +| **anBattery** (analysis affix-template battery) | **51.4%** | +| anCascade (analysis mrule cascade) | 18.4% | +| anOther (anTotal residual) | 5.1% | +| anPhono | 0.0% | +| synCascade + synBattery + synForward + lookup | ~5.0% | +| unaccounted | 20.1% | + +Against Amharic, where `anCascade` is ~95% of `anTotal` and `anBattery` only 4%. **So there is no +single hot spot across grammars — only a single hot *phase*.** Sena is template-battery-bound; +Amharic is cascade-bound; neither is synthesis-bound. An earlier version of this section claimed +the cascade was "the target" on the strength of Amharic alone. That was the Sena-shaped error in +reverse, caught within one run. + +Two things worth carrying forward: + +- **The template battery is still 51.4% of Sena after being memoized.** Phase 3b measured it at + 93% pre-memo and its memo bought a 5x. It remains the largest single bucket. The memo reduced + how often the battery runs; it did not reduce what a run costs. +- **The common thread is per-node cost, not node count.** Amharic spends ~170 ms per analysis + state in a cascade already at its state floor. Every optimization attempted in this area — + memoization, key narrowing, lexical gating, tandem intersection — has reduced *how many* nodes + are visited. **None has touched what a node costs.** That is the unexplored axis. + +#### One honest gap + +Sena's `unaccounted` is **20.1%** (24.0% on `cinacemerwa`, 7.6% on `atawirambo`) — Amharic's is +0.1%, so this is Sena-specific, not a broken bracket. The grounded hypothesis is +`Word.ExpandAlternatives()` (`Word.cs:470`), called per synthesis word in `SynthesizeSequential` +outside every timed region, doing `Clone`/`Unify`/`Subtract`/`Freeze` work per call. It scales +with the number of analysis/alternative pairs, which fits the per-word spread. **This is a +hypothesis, not a measurement** — one more bracket would settle it, and it should be settled +before anyone quotes the Sena split as complete. + +### 6.5 N1 — the gap is `ExpandAlternatives`, confirmed; a second, larger build candidate + +**Hypothesis confirmed.** With `synExpand` broken out as its own exclusive bracket around +`Word.ExpandAlternatives()` in `Morpher.SynthesizeSequential` (the only call site this harness +reaches -- the harness always calls `ParseWord` with `guessRoot: false`, so the `guessRoot` +`ExpandAlternatives` call at the time was never in scope), Sena's `unaccounted` collapses from +20.1% to **1.5% pooled**, and per word: `atawirambo` 7.6% → **0.77%**, `kukucitirani` → **1.19%**, +`cinacemerwa` 24.0% → **2.01%**. All comfortably under the 5% gate. `synExpand` itself absorbs +almost exactly what `unaccounted` lost: **20.3% of wall pooled** (7.9% / 17.5% / 26.4% per word). +Determinism violations: 0. Full suite reconfirmed green (582/1/0) with the new brackets in place. + +| word | wall | synExpand | unaccounted (old → new) | +| --- | --- | --- | --- | +| `atawirambo` | 12,101 ms | 950 ms (7.9%) | 7.6% → **0.77%** | +| `kukucitirani` | 51,872 ms | 9,087 ms (17.5%) | (n/a) → **1.19%** | +| `cinacemerwa` | 48,740 ms | 12,877 ms (26.4%) | 24.0% → **2.01%** | +| **pooled** | 112,713 ms | 22,914 ms (20.3%) | 20.1% → **1.5%** | + +**This does not flip the 6.4 fold-sharing verdict for parsing.** `ExpandAlternatives` runs before +`_synthesisRule.Apply`, i.e. outside the fold P1c measures -- the fold-step ratio recomputed on +this same run is **265.52x**, matching the earlier 265.5x, and `synCascade + synBattery + +synForward` is still ~5% of wall (6,017 ms / 112,713 ms = 5.34% pooled, ceiling `5.34% x (1 - +1/265.52)` = **5.32%**, consistent with 6.4's 4.98%). The do-not-build verdict for the fold-step +build stands, unchanged, for both grammars. + +**But the dedupe census at fold entry finds a second, distinct, and larger opportunity.** For +every `Word` `ExpandAlternatives()` produces, right before it enters `_synthesisRule.Apply`: + +| word | alternatives | distinct | distinct/total | dupe: same analysis word | dupe: different | +| --- | --- | --- | --- | --- | --- | +| `atawirambo` | 17,699 | 12 | 0.07% | 17,380 (**98.3%**) | 307 (1.7%) | +| `kukucitirani` | 158,480 | 18 | 0.01% | 158,462 (**100%**) | 0 (0%) | +| `cinacemerwa` | 218,847 | 31 | 0.01% | 159,747 (**73.0%**) | 59,069 (27.0%) | +| **pooled** | 395,026 | 61 | **0.02%** | 335,589 (**85.0%**) | 59,376 (15.0%) | + +**Gate: ON.** `distinct/total` (0.02% pooled, worst case 0.07%) is far under the 0.2 threshold, +and the same-analysis-word share of duplicates is the majority on every word individually (73.0% +- 100%) as well as pooled (85.0%) -- the OFF condition (duplicates predominantly cross-word) never +fires anywhere in this data. Most of the redundancy `ExpandAlternatives` re-does (`Clone` + +`Unify` + `Subtract` + `Freeze`, per Word.cs:470) is re-deriving an alternative already produced +earlier in the *same* analysis word's expansion, which means it is interceptable with a +per-analysis-word fingerprint cache checked before that work runs, not just after. + +Ceiling if the same-word share were fully intercepted (`synExpand` share x same-word fraction of +alternatives, as a first-order estimate assuming roughly uniform per-alternative cost): pooled +20.3% x 85.0% ≈ **17.3% of Sena wall time** -- an order of magnitude above the fold-step build's +~5% ceiling, and a real, newly-identified target. `cinacemerwa`'s 27% cross-word remainder is the +honest caveat: even a perfect pre-expansion dedupe leaves a residual only a post-expansion +fingerprint (the existing P1c one) could catch, and that residual is real (59,069 events on this +one word alone). + +**This is a measurement, not a build.** N1's brief was to settle the hypothesis and census, not to +implement the cache; it is reported here as a scoped, gated, next candidate distinct from the +fold-step build 6.4 already closed. + +### 6.6 Follow-on notes + +- **Cost-weight P1b.** Count is not cost. Attribute wall time, not events, to each die point. +- **The trail-position finding needs its own look.** If `RuleNotApplicableOrPatternMismatch` is + dominated by the synthesis cascade trying every rule at each node when the trail dictates exactly + one pending rule, that is the "~40x free" observation already recorded in + `docs/hermitcrab-parse-algorithm-analysis.md` (complexity-cap branch), independently + reconfirmed here across two typologies. Indexing synthesis rules by trail position is a much + smaller change than anything else in this plan. Cheap to measure, cheap to build. + + +--- + +## 7. N1 correction — the census ratio repeats a retracted measurement + +The N1 hypothesis result stands and is valuable. The N1 *census* conclusion does not, and the +"build is ON, ceiling 17.3%" line in section 6.5 must not be cited. + +### What stands + +`unaccounted` 20.1% -> **1.5%** pooled once `synExpand` is broken out, which absorbed 20.3%. +**`Word.ExpandAlternatives` is 20.3% of Sena wall time**, invisible to every prior round of +instrumentation. Census denominators cross-check exactly against an independent earlier run +(`cinacemerwa` 218,847, `atawirambo` 17,699, `kukucitirani` 158,480). Determinism violations: 0. + +### What does not + +The census reports 395,026 alternatives collapsing to **61 distinct** (0.02%), giving a claimed +~17.3% ceiling. That ratio is measured with the P1c fingerprint, which carries +`PendingTrailPosition` — an integer index — and **no remaining-trail content** +(`FingerprintHash`/`FingerprintEquals`, verified). For a fold *step* that is defensible because +the continuation is re-anchored. For **fold-entry dedupe**, which is what the census measures, +skipping an alternative because another shares its fingerprint discards its distinct +continuation: lost parses. Completeness is the one thing that is never negotiable here. + +**This exact measurement has been made before**, at the same `ExpandAlternatives` call sites, with +the same `cinacemerwa` denominator of 218,847, on branch `parse-forest-tandem`: + +| probe | key | ratio | +| --- | --- | --- | +| F1 v1 | naive | **9,774x** — proven unsound, 4/30 groups produced different outputs | +| F1 v2 | + pending-rule multiset | 28.72x — 2 residual violations, genuine non-commutativity | +| F2 as shipped | fully order-sound | **15–40% call reduction** | + +The 6,476x is the 9,774x again. The adversarial review predicted this failure mode by name one +step earlier — trail position without trail content — and the probe used that fingerprint anyway. + +### Corrected ceiling + +**20.3% x (15–40% sound dedupe) ≈ 3–8% of Sena wall time.** + +Still the largest single opportunity this effort has surfaced, and on an axis nobody had +instrumented. But a fifth of the claimed figure, and it needs a trail-complete key before any +build. The same-analysis-word share (85% pooled) is the encouraging part: alternatives from one +analysis word share a trail, so for *those* the existing fingerprint may already be adequate — +that, not the headline ratio, is the thing worth re-measuring with a sound key. + + +--- + +## 8. The corrected per-fixture table — the size argument is refuted + +Re-run with the correct denominator (`synTotalShare = (synCascade + synBattery + synForward) / wall`) +and a reliability flag (`wallMs >= 50`). **This overturns section 6.4's size-vs-typology argument.** + +Reliable rows only (`wallMs >= 50`), sorted by fold-step ceiling: + +| fixture | ratio | synTotalShare | **foldStepCeiling** | wallMs | words | +| --- | --- | --- | --- | --- | --- | +| edge-cases/deep-optional-affix-nesting | 3.22x | **73.1%** | **50.4%** | 2,839 | 3 | +| languages/suffixing-evidential-adjacency-chain | 8.10x | **59.3%** | **52.0%** | 107 | 28 | +| languages/suffixing-extension-slot-ordering | 2.15x | 24.7% | 13.2% | 69 | 53 | +| languages/fusional-realizational-morphology | 1.48x | 14.3% | 4.6% | 93 | 63 | + +**Two reliable fixtures show a ~50% wall-clock ceiling — a 2x speedup.** + +### What was wrong before + +Section 6.4 argued that fixtures only look synthesis-heavy because they are *small*, citing +`deep-optional-affix-nesting` as the largest fixture with the lowest synthesis share (6.1%) among +the high-ratio group. **That 6.1% was `forwardShare` — the wrong denominator.** Its correct +`synTotalShare` is **73.1%**, the highest of any reliable row, and it is by far the largest and +most trustworthy timing sample in the set (2.8 s, versus everything else under 110 ms). + +So the trend the size argument rested on does not exist. The largest, most reliable fixture is +the *most* synthesis-bound, not the least. The argument was an artifact of the same wrong-share +error corrected in section 6.4, and it should not have been used to discount the fixture evidence. + +### Bonus finding in the same table + +`deep-optional-affix-nesting`: `altTotal` 926 -> `altDistinct` 3 (**0.32%**), with +`dupeSameWordPct` = **100.0%**. Every duplicate traces to the same analysis word, therefore shares +a trail — which is exactly the case where the existing fingerprint is adequate, because what it +omits (remaining-trail content) is identical across the group. The `ExpandAlternatives` dedupe is +fully interceptable pre-expansion on this grammar. + +### Categories that clear the 2x bar + +| group | fixtures at >=2x | verdict | +| --- | --- | --- | +| **A. Affix template slots** (obligatory/disjunctive/ordering) | 5 of 6 — 8.10x, 3.22x, 3.00x, 2.81x, 2.15x | **Clears, with both reliable rows** | +| **B. Disjunctive allomorphs / free fluctuation** | 2 of 2 — 3.00x, 2.00x | Clears; small samples, coherent mechanism | +| **C. NaturalClass precision** | 1 of 1 — 3.94x (81.1% share) | Clears; single fixture, 11.9 ms | +| **D. Stem names** | 1 of 1 — 2.00x (61.0% share) | Clears; single fixture, 2.1 ms | +| E. Subrule gating | 2.00x from **2 applications** | Discarded — one observation | +| F–K (compounding, MPR, metathesis/truncation, rewrite, loader, feature breadth) | none | Fail; retained as evidence for future grammars | + +Groups B, C and D rest on sub-12 ms fixtures and cannot carry a timing claim on their own. Their +value is that the mechanism *engages* for those constructs — the wall-clock case is carried by +group A, where both reliable rows sit. + + +--- + +## 9. Reproducibility: two independent runs + +The fixture pass was run twice, by two operators, on the same code. This is the check that decides +which rows can carry a claim. + +| fixture | run A ceiling | run B ceiling | reliable | +| --- | --- | --- | --- | +| edge-cases/deep-optional-affix-nesting | 50.41% | **50.94%** | yes (2.4–2.8 s) | +| languages/suffixing-evidential-adjacency-chain | 51.95% | **50.23%** | yes (96–107 ms) | +| languages/suffixing-extension-slot-ordering | 13.17% | 10.16% | yes (62–69 ms) | +| edge-cases/strrep-identity — `synTotalShare` | 81.1% | **65.8%** | no (9.7–11.9 ms) | +| edge-cases/strrep-identity — `synExpandShare` | 1.3% | **14.5%** | no | + +**The two reliable ≥2x rows reproduce to within ~1.7 points, both at ~50%.** That is the +load-bearing result and it is stable across runs. + +**The unreliable rows do not reproduce.** `strrep-identity`'s `synExpandShare` moved 1.3% -> 14.5%, +an 11x swing on a 10 ms fixture. This is direct evidence for the `wallMs >= 50` flag rather than an +assumption behind it: sub-50 ms fixtures cannot carry a timing claim, and groups C and D (both +single sub-12 ms fixtures) must be read as "the mechanism engages for this construct", never as a +speedup estimate. + +Deterministic counters — `applications`, `distinct`, `ratio`, `altTotal`, `altDistinct` — were +byte-identical across both runs, as they must be. Determinism violations: 0 in both. + +### Commit hygiene note + +Commit `1a7d484c` bundles the harness reporting change with the section 8 analysis, because the +agent that wrote the reporting change stopped with it uncommitted and the work was swept in when +the analysis was committed. Two concerns in one commit; flagged here rather than rewritten, since +the PR body is the review artifact. diff --git a/src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs b/src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs index e4d223bd..146810be 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs @@ -121,6 +121,7 @@ internal bool IsWordValid(Morpher morpher, Word word) Environments ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.AllomorphEnvironment); return false; } @@ -147,6 +148,9 @@ internal bool IsWordValid(Morpher morpher, Word word) disjunctiveAllomorph ); } + // Grouped with the Environments check above under "allomorph environment": this is + // still deciding which allomorph's environment wins, not a co-occurrence rule. + SynthesisProbe.RecordDie(SynthesisDiePoint.AllomorphEnvironment); return false; } } @@ -173,6 +177,7 @@ protected virtual bool CheckAllomorphConstraints(Morpher morpher, Allomorph allo rule ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.IsWordValid); return false; } } @@ -195,6 +200,7 @@ protected virtual bool CheckAllomorphConstraints(Morpher morpher, Allomorph allo rule ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.IsWordValid); return false; } } diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index 48bc2474..982fe9fb 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -125,7 +125,19 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) input = input.Clone(); input.Stratum = _stratum; - _prulesRule.Apply(input); + // AnPhonoTicks: the analysis phonological-rule cascade for this stratum. One call per + // AnalysisStratumRule.Apply invocation (i.e. per stratum transition), not recursive -- unlike + // the mrule cascade/template battery below, this call never re-enters AnalysisStratumRule. + if (SynthesisProbe.Enabled) + { + long phonoStart = Stopwatch.GetTimestamp(); + _prulesRule.Apply(input); + SynthesisProbe.AddAnPhonoTicks(Stopwatch.GetTimestamp() - phonoStart); + } + else + { + _prulesRule.Apply(input); + } input.Freeze(); IDictionary shapeWord = null; // Don't merge if tracing because it messes up the tracing. @@ -170,7 +182,29 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) private IEnumerable ApplyMorphologicalRules(Word input) { - foreach (Word mruleOutWord in _mrulesRule.Apply(input).Distinct(FreezableEqualityComparer.Default)) + // AnCascadeTicks: the analysis morphological-rule cascade entry point. This is the whole + // MemoizedCombinationRuleCascade (or PermutationRuleCascade/ParallelCombinationRuleCascade, + // depending on stratum.MorphologicalRuleOrder and Morpher.MaxDegreeOfParallelism) for this + // node -- its own internal recursion (e.g. MemoizedCombinationRuleCascade.ApplyRules) happens + // inside this one call, including its memo lookups/writes, so it is captured with no double + // counting against AnBatteryTicks below, which brackets a disjoint call (ApplyTemplateBattery + // never calls back into this cascade). Materialized only when the probe is on, same as the + // synthesis-side SynthesisStratumRule.ApplyMorphologicalRules -- the same results are yielded + // either way. + IEnumerable mruleOutWords; + if (SynthesisProbe.Enabled) + { + long start = Stopwatch.GetTimestamp(); + var materialized = _mrulesRule.Apply(input).ToList(); + SynthesisProbe.AddAnCascadeTicks(Stopwatch.GetTimestamp() - start); + mruleOutWords = materialized; + } + else + { + mruleOutWords = _mrulesRule.Apply(input); + } + + foreach (Word mruleOutWord in mruleOutWords.Distinct(FreezableEqualityComparer.Default)) { switch (_stratum.MorphologicalRuleOrder) { @@ -218,7 +252,25 @@ private IEnumerable ApplyTemplateBattery(Word input) private IEnumerable ApplyTemplates(Word input) { - foreach (Word tempOutWord in ApplyTemplateBattery(input).Distinct(FreezableEqualityComparer.Default)) + // AnBatteryTicks: the analysis affix-template battery entry point (ApplyTemplateBattery), which + // internally either replays a memo hit or runs the full RuleBatch and stores it -- either way + // captured here with no double counting against AnCascadeTicks (ApplyTemplateBattery never + // calls back into the mrule cascade). Materialized only when the probe is on, mirroring + // SynthesisStratumRule.ApplyTemplates. + IEnumerable templateOutWords; + if (SynthesisProbe.Enabled) + { + long start = Stopwatch.GetTimestamp(); + var materialized = ApplyTemplateBattery(input).ToList(); + SynthesisProbe.AddAnBatteryTicks(Stopwatch.GetTimestamp() - start); + templateOutWords = materialized; + } + else + { + templateOutWords = ApplyTemplateBattery(input); + } + + foreach (Word tempOutWord in templateOutWords.Distinct(FreezableEqualityComparer.Default)) { switch (_stratum.MorphologicalRuleOrder) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 54ac7b5e..6d756369 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -175,7 +176,26 @@ public IEnumerable ParseWord(string word, out object trace, bool guessRoot trace = input.CurrentTrace; // Unapply rules - var analyses = new ConcurrentQueue(_analysisRule.Apply(input)); + // AnTotalTicks is the outer/nested bucket for the whole analysis phase (P1a follow-up, see + // SynthesisProbe's wall-time-split remarks): it brackets this one call, which -- via + // AnalysisLanguageRule.Apply -- recurses through every stratum's AnalysisStratumRule.Apply, + // inside which AnCascadeTicks/AnBatteryTicks/AnPhonoTicks accumulate disjoint slices. Those + // three sum to <= this bucket; the remainder is analysis-side orchestration the sub-timers + // don't individually cover (recursion glue, Clone/Freeze, Distinct, memo-key hashing outside + // the cascade/battery calls themselves). + IEnumerable analysisResults; + if (SynthesisProbe.Enabled) + { + long anStart = Stopwatch.GetTimestamp(); + var materialized = _analysisRule.Apply(input).ToList(); + SynthesisProbe.AddAnTotalTicks(Stopwatch.GetTimestamp() - anStart); + analysisResults = materialized; + } + else + { + analysisResults = _analysisRule.Apply(input); + } + var analyses = new ConcurrentQueue(analysisResults); if (scope != null) AccumulateMemoDiagnostics(scope); @@ -349,18 +369,81 @@ private IEnumerable SynthesizeSequential(string word, IEnumerable an int alternativeCount = 0; foreach (Word analysisWord in analyses) { - foreach (Word synthesisWord in LexicalLookup(analysisWord)) + // Materialized (rather than left lazy) so the P1a wall-time split can bracket exactly the + // work LexicalLookup itself does, separate from what the caller does with the results. + // Instrumentation only: SynthesisProbe.Enabled is false outside the P1 harness, and this + // still enumerates every result LexicalLookup would have yielded either way, so the set of + // words synthesis proceeds with is unchanged. + List lookups; + if (SynthesisProbe.Enabled) + { + long lookupStart = Stopwatch.GetTimestamp(); + lookups = LexicalLookup(analysisWord).ToList(); + SynthesisProbe.AddLexicalLookupTicks(Stopwatch.GetTimestamp() - lookupStart); + if (lookups.Count == 0) + SynthesisProbe.RecordDie(SynthesisDiePoint.LexicalLookupMiss); + } + else + { + lookups = LexicalLookup(analysisWord).ToList(); + } + + foreach (Word synthesisWord in lookups) { - foreach (Word alternative in synthesisWord.ExpandAlternatives()) + // N1 (docs/hermitcrab-synthesis-fold-probes.md section 6.4's "one honest gap"): + // ExpandAlternatives() itself is bracketed here as its own exclusive top-level slice, so + // the recursive Clone/Unify/Subtract/Freeze work it does is no longer part of + // "unaccounted". Materialized either way (ExpandAlternatives already returns an IList, + // not lazy), so this changes no behaviour when the probe is disabled. + IList alternatives; + if (SynthesisProbe.Enabled) + { + long expandStart = Stopwatch.GetTimestamp(); + alternatives = synthesisWord.ExpandAlternatives(); + SynthesisProbe.AddSynExpandTicks(Stopwatch.GetTimestamp() - expandStart); + } + else + { + alternatives = synthesisWord.ExpandAlternatives(); + } + + foreach (Word alternative in alternatives) { alternativeCount++; if (MaxAlternatives > 0 && alternativeCount > MaxAlternatives) throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); + + if (!SynthesisProbe.Enabled) + { + foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) + { + if (IsMatch(word, validWord)) + matches.Add(validWord); + } + continue; + } + + // N1 dedupe census: this alternative is about to enter the fold + // (_synthesisRule.Apply below). analysisWord is the outer loop's analysis word -- + // exactly the provenance identity the census needs, already in scope. + SynthesisProbe.RecordFoldEntry(analysisWord, alternative); + + // P1a's "forward synthesis" bucket is this call's wall time minus whatever the + // cascade/template-battery timers (accumulated separately inside SynthesisStratumRule) + // recorded during it -- the residual is _synthesisRule.Apply's own orchestration + // plus IsWordValid and IsMatch, exactly as the plan defines the bucket. + long cascadeBefore = SynthesisProbe.SynCascadeTicks; + long batteryBefore = SynthesisProbe.SynBatteryTicks; + long forwardStart = Stopwatch.GetTimestamp(); foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) { if (IsMatch(word, validWord)) matches.Add(validWord); } + long forwardTotal = Stopwatch.GetTimestamp() - forwardStart; + long cascadeDelta = SynthesisProbe.SynCascadeTicks - cascadeBefore; + long batteryDelta = SynthesisProbe.SynBatteryTicks - batteryBefore; + SynthesisProbe.AddSynForwardTicks(forwardTotal - cascadeDelta - batteryDelta); } } } @@ -640,6 +723,7 @@ private bool IsWordValid(Word word) { if (_traceManager.IsTracing) _traceManager.Failed(_lang, word, FailureReason.PartialParse, null, null); + SynthesisProbe.RecordDie(SynthesisDiePoint.IsWordValid); return false; } @@ -654,9 +738,13 @@ private bool IsWordValid(Word word) { if (_traceManager.IsTracing) _traceManager.Failed(_lang, word, FailureReason.ObligatorySyntacticFeatures, null, feature); + SynthesisProbe.RecordDie(SynthesisDiePoint.IsWordValid); return false; } + // Sub-reasons (allomorph Environments vs. co-occurrence rules) are recorded inside + // Allomorph.IsWordValid itself, at the die point that actually fired -- not duplicated here, + // since a single word can carry several allomorphs and only some may reject. return word.Allomorphs.All(allo => allo.IsWordValid(this, word)); } @@ -672,6 +760,7 @@ private bool IsMatch(string word, Word validWord) { _traceManager.Failed(_lang, validWord, FailureReason.SurfaceFormMismatch, null, word); } + SynthesisProbe.RecordDie(SynthesisDiePoint.SurfaceFormMismatch); return false; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs index f7dc9c0d..73d079ab 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs @@ -41,7 +41,10 @@ public SynthesisAffixProcessRule(Morpher morpher, AffixProcessRule rule) public IEnumerable Apply(Word input) { if (!input.IsMorphologicalRuleApplicable(_rule)) + { + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); return Enumerable.Empty(); + } if (input.GetApplicationCount(_rule) >= _rule.MaxApplicationCount) { @@ -55,6 +58,7 @@ public IEnumerable Apply(Word input) _rule.MaxApplicationCount ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.ApplicationCount); return Enumerable.Empty(); } @@ -77,6 +81,7 @@ public IEnumerable Apply(Word input) null ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); return Enumerable.Empty(); } @@ -100,6 +105,7 @@ public IEnumerable Apply(Word input) null ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); return Enumerable.Empty(); } @@ -115,6 +121,7 @@ public IEnumerable Apply(Word input) _rule.RequiredStemName ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); return Enumerable.Empty(); } @@ -131,6 +138,7 @@ public IEnumerable Apply(Word input) _rule.RequiredSyntacticFeatureStruct ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.FeatureUnification); return Enumerable.Empty(); } @@ -155,6 +163,7 @@ public IEnumerable Apply(Word input) group ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.MprFeatures); continue; } if ( @@ -172,6 +181,7 @@ public IEnumerable Apply(Word input) group ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.MprFeatures); continue; } @@ -226,12 +236,31 @@ public IEnumerable Apply(Word input) break; } } - else if (_morpher.TraceManager.IsTracing) + else { - _morpher.TraceManager.MorphologicalRuleNotApplied(_rule, i, input, FailureReason.Pattern, null); + if (_morpher.TraceManager.IsTracing) + { + _morpher.TraceManager.MorphologicalRuleNotApplied( + _rule, + i, + input, + FailureReason.Pattern, + null + ); + } + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); } } + // Recorded once for the whole call, not per allomorph: several allomorphs of the same _rule + // can legitimately all pattern-match the same input before the disjunctive-environment break + // above, so (fingerprint, rule) is properly a set-valued fold step here -- exactly the shape + // the plan doc's second trap already calls out for realizational rules ("any stored partial + // must be a set, like MemoEntry.Results, not a value"). Recording per allomorph instead would + // make ordinary disjunctive fan-out look like a determinism violation. + if (output.Count > 0) + SynthesisProbe.RecordApplications(input, _rule, output); + return output; } } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs index 061ae906..5e72e2e6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs @@ -58,11 +58,15 @@ public IEnumerable Apply(Word input) 1 ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.ApplicationCount); return Enumerable.Empty(); } if (!_rule.RealizationalFeatureStruct.Subsumes(input.RealizationalFeatureStruct)) + { + SynthesisProbe.RecordDie(SynthesisDiePoint.RealizationalSubsumptionOrBlocking); return Enumerable.Empty(); + } if ( !_rule.RealizationalFeatureStruct.IsEmpty @@ -73,6 +77,7 @@ public IEnumerable Apply(Word input) ) ) { + SynthesisProbe.RecordDie(SynthesisDiePoint.RealizationalSubsumptionOrBlocking); return Enumerable.Empty(); } @@ -89,6 +94,7 @@ public IEnumerable Apply(Word input) _rule.RequiredSyntacticFeatureStruct ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.FeatureUnification); return Enumerable.Empty(); } @@ -113,6 +119,7 @@ public IEnumerable Apply(Word input) group ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.MprFeatures); continue; } if ( @@ -130,6 +137,7 @@ public IEnumerable Apply(Word input) group ); } + SynthesisProbe.RecordDie(SynthesisDiePoint.MprFeatures); continue; } @@ -174,12 +182,28 @@ public IEnumerable Apply(Word input) break; } } - else if (_morpher.TraceManager.IsTracing) + else { - _morpher.TraceManager.MorphologicalRuleNotApplied(_rule, i, input, FailureReason.Pattern, null); + if (_morpher.TraceManager.IsTracing) + { + _morpher.TraceManager.MorphologicalRuleNotApplied( + _rule, + i, + input, + FailureReason.Pattern, + null + ); + } + SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); } } + // Recorded once for the whole call, not per allomorph -- see the matching comment in + // SynthesisAffixProcessRule.Apply: several allomorphs can legitimately all pattern-match one + // input, so (fingerprint, rule) is a set-valued fold step here. + if (output.Count > 0) + SynthesisProbe.RecordApplications(input, _rule, output); + return output; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/RuleLengthClassifier.cs b/src/SIL.Machine.Morphology.HermitCrab/RuleLengthClassifier.cs new file mode 100644 index 00000000..be0b90f8 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RuleLengthClassifier.cs @@ -0,0 +1,178 @@ +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// What un-applying a morphological rule does to the length of the word, decided statically from the + /// rule's own declaration (hermitcrab-forest-memo-plan.md Stage 1). + /// + internal enum UnapplicationLengthEffect + { + /// + /// Every un-application strictly shortens the word. Such a rule cannot drive an infinite regress -- + /// shape length is already a decreasing measure -- so it does not need to be counted in + /// to guarantee termination. + /// + Shrinking, + + /// + /// Some un-application leaves the word the same length or longer (a zero morpheme, or a rule that + /// deletes material on synthesis and therefore restores it on analysis). These are exactly the + /// rules that can loop, so they stay in the key. + /// + NonShrinking, + + /// + /// Not decidable by this classifier. Treated exactly like by the key -- + /// kept distinct only so a grammar census can tell "this grammar has zero morphemes" apart from + /// "this grammar has constructs we cannot analyse." + /// + Unknown + } + + /// + /// Classifies each morphological rule by so that + /// can drop the per-rule un-application counts that do no work. + /// + /// The direction is the thing to keep straight: on the ANALYSIS side rules are un-applied, so an + /// affix rule that INSERTS material on synthesis REMOVES it on analysis (shrinking), while a rule that + /// DELETES material on synthesis RESTORES it on analysis (growing). Getting this backwards would + /// silently drop the rules that actually need counting. + /// + /// Conservative by construction: is the safe answer, + /// because retaining a rule in the key is the status quo and always sound. Only a rule proved to + /// shorten the word on every possible un-application is reported as + /// . + /// + internal static class RuleLengthClassifier + { + public static UnapplicationLengthEffect Classify(IMorphologicalRule rule) + { + switch (rule) + { + case AffixProcessRule affixRule: + return ClassifyAllomorphs(affixRule.Allomorphs); + case RealizationalAffixProcessRule realizationalRule: + return ClassifyAllomorphs(realizationalRule.Allomorphs); + + // A compounding rule's un-application splits one word into a head and a non-head. The head + // shortens, but no material is destroyed, and the interesting bound is MaxStemCount rather + // than length. Word.NonHeadCount is already a key component in its own right, so there is + // nothing to gain by reasoning harder here. + case CompoundingRule _: + return UnapplicationLengthEffect.Unknown; + + default: + return UnapplicationLengthEffect.Unknown; + } + } + + /// + /// Which rules must stay in 's count multiset, built once per + /// grammar rather than once per key: key construction is on the hottest path in the engine, so + /// this must not become a per-key filter. Absent from the map means "retain" -- a rule reached + /// through a path this walk does not cover is retained, never dropped. + /// + public static IReadOnlyDictionary BuildRetainInKeyMap(Language language) + { + var map = new Dictionary(); + foreach (Stratum stratum in language.Strata) + { + foreach (IMorphologicalRule rule in stratum.MorphologicalRules) + map[rule] = Classify(rule) != UnapplicationLengthEffect.Shrinking; + } + return map; + } + + private static UnapplicationLengthEffect ClassifyAllomorphs(IEnumerable allomorphs) + { + bool any = false; + var effect = UnapplicationLengthEffect.Shrinking; + foreach (AffixProcessAllomorph allomorph in allomorphs) + { + any = true; + UnapplicationLengthEffect allomorphEffect = ClassifyAllomorph(allomorph); + if (allomorphEffect == UnapplicationLengthEffect.Unknown) + return UnapplicationLengthEffect.Unknown; + if (allomorphEffect == UnapplicationLengthEffect.NonShrinking) + effect = UnapplicationLengthEffect.NonShrinking; + } + // A rule with no allomorphs cannot be reasoned about; it also cannot un-apply, but say Unknown + // rather than assert that. + return any ? effect : UnapplicationLengthEffect.Unknown; + } + + /// + /// One allomorph shrinks on un-application iff the analysis output is strictly shorter than the + /// analysis input. is the authority on what that + /// output is: it walks the allomorph's Lhs parts and, for each, either copies the span captured by + /// the corresponding Rhs / (length + /// preserved) or calls Untruncate to regenerate the part from its pattern (length ADDED -- + /// this is synthesis-side deletion showing up as analysis-side growth). Meanwhile the Rhs's + /// / material is matched away by the + /// analysis pattern and does not reach the output (length removed). + /// + /// So: shrinking iff every Lhs part is captured, at least one segment is inserted, and no part is + /// copied more than once. + /// + private static UnapplicationLengthEffect ClassifyAllomorph(AffixProcessAllomorph allomorph) + { + int inserted = 0; + var capturedParts = new HashSet(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + switch (action) + { + case InsertSegments insertSegments: + inserted += SegmentCount(insertSegments.Segments.Shape); + break; + case InsertSimpleContext _: + inserted += 1; + break; + case CopyFromInput copyFromInput: + // Reduplication: the same part copied twice. AnalysisMorphologicalTransform emits + // only ONE instance, so this does in fact shrink -- but the un-application is + // nondeterministic in ways this walk does not model, so take the safe answer. + if (!capturedParts.Add(copyFromInput.PartName)) + return UnapplicationLengthEffect.Unknown; + break; + case ModifyFromInput modifyFromInput: + if (!capturedParts.Add(modifyFromInput.PartName)) + return UnapplicationLengthEffect.Unknown; + break; + default: + return UnapplicationLengthEffect.Unknown; + } + } + + // An Lhs part with no corresponding Rhs copy is truncated on synthesis and untruncated on + // analysis. Untruncate adds a node per segment Constraint in the part's pattern, so the + // un-applied word GROWS. (A part whose pattern contains no segment constraints would add + // nothing, but proving that is a refinement this classifier does not need: report + // NonShrinking, which is safe.) + foreach (Pattern part in allomorph.Lhs) + { + if (!capturedParts.Contains(part.Name)) + return UnapplicationLengthEffect.NonShrinking; + } + + // No material inserted on synthesis means nothing removed on analysis: a zero morpheme. These + // are exactly the rules that can produce the N -> V -> N cycle, so they must stay in the key. + return inserted > 0 ? UnapplicationLengthEffect.Shrinking : UnapplicationLengthEffect.NonShrinking; + } + + private static int SegmentCount(Shape shape) + { + int count = 0; + foreach (ShapeNode node in shape) + { + if (node.Annotation.Type() == HCFeatureSystem.Segment) + count++; + } + return count; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/SynthesisProbe.cs b/src/SIL.Machine.Morphology.HermitCrab/SynthesisProbe.cs new file mode 100644 index 00000000..c9fa2495 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/SynthesisProbe.cs @@ -0,0 +1,539 @@ +using System; +using System.Collections.Generic; +using System.Threading; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// The die point a candidate that entered forward synthesis was rejected at, for the P1b histogram + /// (docs/hermitcrab-synthesis-fold-probes.md section 3). Named after the table in that section rather + /// than directly: several values collapse into + /// one die point here because the plan's table groups them (e.g. a rule's own applicability gate and + /// its pattern-match failure are one row, "morphological rule not applicable / pattern match + /// failure"), and two sites -- the mrule-trail gate at SynthesisAffixProcessRule.cs:43 and + /// realizational subsumption/blocking -- have no at all today because they + /// return empty without ever calling (that call is gated on + /// IsTracing, which the memoized synthesis path this probe measures always runs with off). + /// + internal enum SynthesisDiePoint + { + LexicalLookupMiss, + ApplicationCount, + RuleNotApplicableOrPatternMismatch, + AllomorphEnvironment, + RealizationalSubsumptionOrBlocking, + FeatureUnification, + MprFeatures, + IsWordValid, + SurfaceFormMismatch, + } + + /// + /// Static instrumentation hub for the P1 synthesis-fold probes (P1a wall-time split, P1b die-point + /// histogram, P1c fold-step fingerprint ratio; see docs/hermitcrab-synthesis-fold-probes.md section 3). + /// Measurement only: every counter here is write-only from the engine's point of view, and + /// is the single gate that decides whether any of it runs at all -- nothing + /// downstream of that flag ever feeds back into which analyses or syntheses a parse returns, so no + /// engine control flow depends on it. Defaults to false so the ordinary test suite (several + /// non-Explicit tests construct a sequential Morpher, e.g. MorpherTests, AnalysisStratumRuleTests) never + /// pays for the P1c fold-step table, which pins Word references and would otherwise grow for the whole + /// process lifetime with nothing ever the wiser to reset it. Only the P1 harness + /// (SynthesisFoldProbe.cs, [Explicit]) turns this on. + /// + /// Not lock-free: the harness always runs the sequential (maxDegreeOfParallelism: 1) path one word at a + /// time, but the locks below are cheap insurance against NUnit fixture-level parallelism sharing this + /// process, not a performance-sensitive path. + /// + /// + internal static class SynthesisProbe + { + internal static volatile bool Enabled; + + // ---- P1a: wall-time split ---- + // Two sides, kept as separate labelled buckets per the P1a follow-up (docs/hermitcrab-synthesis-fold-probes.md + // section 3): the "syn*" buckets bracket disjoint regions inside Morpher.Synthesize/SynthesisStratumRule + // (unchanged from the original P1a cut, just renamed so they read unambiguously next to the analysis + // side). The "an*" buckets are the new analysis-side instrumentation this follow-up adds. AnTotalTicks + // is a NESTED/INCLUSIVE total -- it brackets the whole of Morpher.ParseWord's `_analysisRule.Apply(input)` + // call, and AnCascadeTicks/AnBatteryTicks/AnPhonoTicks are disjoint slices taken from calls *within* that + // same call tree (see AnalysisStratumRule), so AnTotalTicks >= AnCascadeTicks + AnBatteryTicks + AnPhonoTicks. + // The three "an*" slice buckets are mutually disjoint from each other and from the "syn*"/lookup buckets, + // by the same non-overlapping-call-site construction the original synthesis-side buckets already used + // (see SynthesisStratumRule's ApplyMorphologicalRules/ApplyTemplates remarks). + private static long _lexicalLookupTicks; + private static long _synCascadeTicks; + private static long _synBatteryTicks; + private static long _synForwardTicks; + private static long _synExpandTicks; + private static long _anTotalTicks; + private static long _anCascadeTicks; + private static long _anBatteryTicks; + private static long _anPhonoTicks; + + internal static long LexicalLookupTicks => Interlocked.Read(ref _lexicalLookupTicks); + internal static long SynCascadeTicks => Interlocked.Read(ref _synCascadeTicks); + internal static long SynBatteryTicks => Interlocked.Read(ref _synBatteryTicks); + internal static long SynForwardTicks => Interlocked.Read(ref _synForwardTicks); + + // ---- N1: ExpandAlternatives bracket ---- + // docs/hermitcrab-synthesis-fold-probes.md section 6.4's "one honest gap": Word.ExpandAlternatives + // (Word.cs:470) is called per synthesis word in Morpher.SynthesizeSequential OUTSIDE every timed + // region that existed before N1, doing recursive Clone/Unify/Subtract/Freeze work per alternative. + // This is a new EXCLUSIVE top-level slice (bracketed around the ExpandAlternatives() call itself, + // not around anything already covered by lookup/synCascade/synBattery/synForward/anTotal), so + // "unaccounted" in SynthesisFoldProbe shrinks by exactly what this bucket gains. + internal static long SynExpandTicks => Interlocked.Read(ref _synExpandTicks); + internal static long AnTotalTicks => Interlocked.Read(ref _anTotalTicks); + internal static long AnCascadeTicks => Interlocked.Read(ref _anCascadeTicks); + internal static long AnBatteryTicks => Interlocked.Read(ref _anBatteryTicks); + internal static long AnPhonoTicks => Interlocked.Read(ref _anPhonoTicks); + + internal static void AddLexicalLookupTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _lexicalLookupTicks, ticks); + } + + internal static void AddSynCascadeTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _synCascadeTicks, ticks); + } + + internal static void AddSynBatteryTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _synBatteryTicks, ticks); + } + + internal static void AddSynForwardTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _synForwardTicks, ticks); + } + + internal static void AddSynExpandTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _synExpandTicks, ticks); + } + + internal static void AddAnTotalTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _anTotalTicks, ticks); + } + + internal static void AddAnCascadeTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _anCascadeTicks, ticks); + } + + internal static void AddAnBatteryTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _anBatteryTicks, ticks); + } + + internal static void AddAnPhonoTicks(long ticks) + { + if (Enabled) + Interlocked.Add(ref _anPhonoTicks, ticks); + } + + internal static void ResetWallTime() + { + Interlocked.Exchange(ref _lexicalLookupTicks, 0); + Interlocked.Exchange(ref _synCascadeTicks, 0); + Interlocked.Exchange(ref _synBatteryTicks, 0); + Interlocked.Exchange(ref _synForwardTicks, 0); + Interlocked.Exchange(ref _synExpandTicks, 0); + Interlocked.Exchange(ref _anTotalTicks, 0); + Interlocked.Exchange(ref _anCascadeTicks, 0); + Interlocked.Exchange(ref _anBatteryTicks, 0); + Interlocked.Exchange(ref _anPhonoTicks, 0); + } + + // ---- P1b: die-point histogram ---- + // Counts rejection EVENTS, not distinct top-level candidates: a single alternative can branch into + // many internal rule/allomorph attempts, each independently able to die at a different check, and + // there is no tractable way to attribute "the" one reason a whole subtree failed without changing + // the traversal. This is what the table in section 3 is measuring anyway -- "which checks kill + // candidates" -- and it is exactly what cinacemerwa's 218,847-candidate figure counts. + private static readonly long[] _diePoints = new long[Enum.GetValues(typeof(SynthesisDiePoint)).Length]; + + internal static void RecordDie(SynthesisDiePoint point) + { + if (Enabled) + Interlocked.Increment(ref _diePoints[(int)point]); + } + + internal static long GetDieCount(SynthesisDiePoint point) => Interlocked.Read(ref _diePoints[(int)point]); + + internal static void ResetDiePoints() + { + for (int i = 0; i < _diePoints.Length; i++) + Interlocked.Exchange(ref _diePoints[i], 0); + } + + // ---- P1c: fold-step fingerprint ratio ---- + // Keyed on the INPUT to a synthesis step (everything a step reads, see FingerprintHash/Equals + // below) plus the rule that was applied. Persists across a whole fixture (not reset per word) so + // the distinct-pair count is a true count over the combined stream, not a sum of per-word counts + // that would double-count a pair recurring across words of the same fixture. + private static readonly Dictionary> _foldSteps = new Dictionary>(); + private static long _totalApplications; + private static long _determinismViolations; + + internal static long TotalApplications => Interlocked.Read(ref _totalApplications); + + internal static long DistinctFoldSteps + { + get + { + lock (_foldSteps) + return _foldSteps.Count; + } + } + + internal static long DeterminismViolations => Interlocked.Read(ref _determinismViolations); + + /// + /// Records one call's worth of successful synthesis morphological-rule/template applications: + /// was applied to , producing every word in + /// . Grouped as one call, not one per output, because several allomorphs + /// of the same rule can legitimately all pattern-match the same input before a disjunctive break -- + /// (fingerprint, rule) is properly a SET-valued fold step, exactly the shape the plan doc's second + /// trap already calls out for realizational rules ("any stored partial must be a set, like + /// MemoEntry.Results, not a value"). Recording per output instead of per set would make ordinary + /// disjunctive fan-out look like a determinism violation on every single-input, multi-allomorph + /// rule -- this was tried first and produced exactly that false-positive flood. + /// + /// The determinism check: if this exact (fingerprint, rule) pair was already seen (from a different + /// call -- a different top-level candidate reaching the same state) with a different OUTCOME SET, + /// that is a fingerprint-completeness bug (see the class remarks on 's callers + /// about the Word.ValueEquals trap this exists to avoid), and is the single most important + /// thing this probe can find. + /// + /// + internal static void RecordApplications(Word input, IMorphologicalRule rule, IReadOnlyList outputs) + { + if (!Enabled || outputs.Count == 0) + return; + + Interlocked.Add(ref _totalApplications, outputs.Count); + var key = new FoldStepKey(input, rule); + lock (_foldSteps) + { + if (_foldSteps.TryGetValue(key, out List priorOutcomes)) + { + if (!OutcomeSetEquals(priorOutcomes, outputs)) + Interlocked.Increment(ref _determinismViolations); + } + else + { + _foldSteps[key] = new List(outputs); + } + } + } + + // Bipartite multiset comparison via FingerprintEquals membership. Outcome sets here are small (at + // most the number of allomorphs one rule declares), so the O(n*m) cost is negligible. + private static bool OutcomeSetEquals(List a, IReadOnlyList b) + { + if (a.Count != b.Count) + return false; + + var matched = new bool[b.Count]; + foreach (Word x in a) + { + bool found = false; + for (int j = 0; j < b.Count; j++) + { + if (!matched[j] && FingerprintEquals(x, b[j])) + { + matched[j] = true; + found = true; + break; + } + } + if (!found) + return false; + } + return true; + } + + internal static void ResetFoldSteps() + { + lock (_foldSteps) + _foldSteps.Clear(); + Interlocked.Exchange(ref _totalApplications, 0); + Interlocked.Exchange(ref _determinismViolations, 0); + ResetFoldEntries(); + } + + // ---- N1: dedupe census at fold entry ---- + // docs/hermitcrab-synthesis-fold-probes.md section "What to build", item 2. For every Word + // ExpandAlternatives() produces that is about to enter _synthesisRule.Apply + // (Morpher.SynthesizeSequential), tracks how many are literal duplicates -- by the SAME P1c + // fingerprint used above, not a second one -- of an earlier alternative, and for each duplicate, + // whether its first occurrence traces back to the same outer analysis word or a different one. + // That split is the decisive one for the gates: same-analysis-word duplication is interceptable + // BEFORE the Clone/Unify/Freeze work ExpandAlternatives just did (a dedupe could sit ahead of + // ExpandAlternatives, keyed on the pre-expansion input); cross-analysis-word duplication is only + // detectable by fingerprinting the post-expansion output, so by the time it is caught the expensive + // work is already spent. + // + // Same persistence lifecycle as _foldSteps (reset together, see ResetFoldSteps): accumulates across + // a whole fixture/corpus so DistinctAlternatives is a true count over the combined stream, not a sum + // of per-word counts that would double-count an alternative recurring across words. + // + private static readonly Dictionary _foldEntries = new Dictionary(); + private static long _totalAlternatives; + private static long _dupeSameAnalysisWord; + private static long _dupeDifferentAnalysisWord; + + internal static long TotalAlternatives => Interlocked.Read(ref _totalAlternatives); + + internal static long DistinctAlternatives + { + get + { + lock (_foldEntries) + return _foldEntries.Count; + } + } + + internal static long DupeSameAnalysisWord => Interlocked.Read(ref _dupeSameAnalysisWord); + internal static long DupeDifferentAnalysisWord => Interlocked.Read(ref _dupeDifferentAnalysisWord); + + /// + /// Records one alternative arriving at the fold entry point (about to be passed to + /// _synthesisRule.Apply), with the outer analysis word whose + /// LexicalLookup/ExpandAlternatives chain produced it. Provenance is decided by + /// reference identity against the analysis word recorded on first sight of this fingerprint -- + /// exactly the loop variable identity Morpher.SynthesizeSequential already has in scope, no + /// separate ID scheme needed. + /// + internal static void RecordFoldEntry(Word analysisWord, Word alternative) + { + if (!Enabled) + return; + + Interlocked.Increment(ref _totalAlternatives); + var key = new AlternativeKey(alternative); + lock (_foldEntries) + { + if (_foldEntries.TryGetValue(key, out Word firstAnalysisWord)) + { + if (ReferenceEquals(firstAnalysisWord, analysisWord)) + Interlocked.Increment(ref _dupeSameAnalysisWord); + else + Interlocked.Increment(ref _dupeDifferentAnalysisWord); + } + else + { + _foldEntries[key] = analysisWord; + } + } + } + + internal static void ResetFoldEntries() + { + lock (_foldEntries) + _foldEntries.Clear(); + Interlocked.Exchange(ref _totalAlternatives, 0); + Interlocked.Exchange(ref _dupeSameAnalysisWord, 0); + Interlocked.Exchange(ref _dupeDifferentAnalysisWord, 0); + } + + internal static void ResetAll() + { + ResetWallTime(); + ResetDiePoints(); + ResetFoldSteps(); + } + + // ---- fingerprint machinery ---- + + /// + /// Everything a synthesis step reads, per docs/hermitcrab-synthesis-fold-probes.md section 3: + /// shape+annotations, syntactic FS, realizational FS, MPR feature set, root allomorph, disjunctive + /// allomorph indices, application counts, IsPartial, IsLastAppliedRuleFinal, stratum, and + /// pending-trail position. Deliberately NOT -- see Word.cs:600's + /// remarks and the plan doc's trap section: that comparer omits SyntacticFeatureStruct, MPR + /// features, and disjunctive allomorph indices, all three of which are checked here precisely + /// because omitting them is what produced an inflated, invalid ratio on the predecessor probe. + /// + /// Field-by-field justification against what Synthesis*.cs actually reads: + /// + /// Shape+annotations () -- every rule's pattern match + /// (SynthesisAffixProcessAllomorphRuleSpec) reads the shape and its morph annotations. + /// SyntacticFeatureStruct -- read by every rule's RequiredSyntacticFeatureStruct.Unify + /// (SynthesisAffixProcessRule.cs, SynthesisRealizationalAffixProcessRule.cs) and by + /// IsUnifiable/RealizationalFeatureStruct checks in SynthesisAffixTemplatesRule. + /// RealizationalFeatureStruct -- read by SynthesisRealizationalAffixProcessRule's Subsumes + /// and IsBlocked checks, and by SynthesisAffixTemplatesRule.ChooseInflectionalStem. + /// MprFeatures -- read by every allomorph's RequiredMprFeatures/ExcludedMprFeatures check in + /// both SynthesisAffixProcessRule and SynthesisRealizationalAffixProcessRule. + /// RootAllomorph -- pattern matching and StemName/environment checks are root-allomorph + /// dependent (SynthesisAffixProcessRule's RequiredStemName check reads + /// input.RootAllomorph.StemName directly). + /// Disjunctive allomorph indices -- read by Allomorph.IsWordValid via + /// GetDisjunctiveAllomorphApplications; two words differing only here can validly disagree on + /// whether a later allomorph is blocked. + /// Application counts -- GetApplicationCount backs every rule's MaxApplicationCount gate + /// (SynthesisAffixProcessRule.cs:46, SynthesisRealizationalAffixProcessRule's "at most once" + /// gate). + /// IsPartial -- read by SynthesisAffixProcessRule's final-template-adjacency gates and by + /// SynthesisAffixTemplatesRule's applicability check. + /// IsLastAppliedRuleFinal -- read by the same final-template-adjacency gates and by + /// SynthesisStratumRule.Apply's own final-rule check. + /// Stratum -- SynthesisStratumRule.Apply gates on + /// input.RootAllomorph.Morpheme.Stratum.Depth > _stratum.Depth, and HasRemainingRulesFromStratum + /// elsewhere reads it. + /// Pending-trail position (PendingTrailPosition / _mruleAppIndex) -- IsMorphologicalRuleApplicable + /// reads exactly this index; paired with the "applied rule" half of the key (see + /// ), which rule was actually attempted is already captured there, so + /// the index alone -- not the full trail content -- is what a step reads before deciding. + /// + /// + /// + private static int FingerprintHash(Word w) + { + int hash = 17; + hash = hash * 31 + w.Shape.GetFrozenHashCode(); + hash = hash * 31 + w.RealizationalFeatureStruct.GetFrozenHashCode(); + hash = hash * 31 + SyntacticFeatureStructWeakHash(w.SyntacticFeatureStruct); + hash = hash * 31 + UnorderedSetHash(w.MprFeatures); + hash = hash * 31 + (w.RootAllomorph?.GetHashCode() ?? 0); + hash = hash * 31 + UnorderedDictHash(w.DisjunctiveAllomorphIndices, UnorderedSetHash); + hash = hash * 31 + UnorderedDictHash(w.AppliedRuleCounts, v => v); + hash = hash * 31 + w.IsPartial.GetHashCode(); + hash = hash * 31 + w.IsLastAppliedRuleFinal.GetHashCode(); + hash = hash * 31 + (w.Stratum?.GetHashCode() ?? 0); + hash = hash * 31 + w.PendingTrailPosition.GetHashCode(); + return hash; + } + + /// + /// The real, exact comparison backing equality (and the determinism + /// check's outcome comparison). is deliberately allowed to be a weak + /// hash for the SyntacticFeatureStruct component (see + /// ) because correctness lives entirely here: a hash + /// collision only costs a linear scan within a bucket, it can never merge two states that this + /// method would call unequal. + /// + private static bool FingerprintEquals(Word a, Word b) + { + if (ReferenceEquals(a, b)) + return true; + if (a == null || b == null) + return false; + + return a.Shape.ValueEquals(b.Shape) + && a.RealizationalFeatureStruct.ValueEquals(b.RealizationalFeatureStruct) + && a.SyntacticFeatureStruct.ValueEquals(b.SyntacticFeatureStruct) + && a.MprFeatures.SetEquals(b.MprFeatures) + && a.RootAllomorph == b.RootAllomorph + && DictEquals(a.DisjunctiveAllomorphIndices, b.DisjunctiveAllomorphIndices, (x, y) => x.SetEquals(y)) + && DictEquals(a.AppliedRuleCounts, b.AppliedRuleCounts, (x, y) => x == y) + && a.IsPartial == b.IsPartial + && a.IsLastAppliedRuleFinal == b.IsLastAppliedRuleFinal + && a.Stratum == b.Stratum + && a.PendingTrailPosition == b.PendingTrailPosition; + } + + // FeatureStruct.GetFrozenHashCode() throws unless the struct is frozen, and Word.Freeze() does NOT + // freeze SyntacticFeatureStruct -- that is exactly why Word.ValueEquals omits it (Word.cs:600). + // Freezing it here as a side effect, even just to hash it, would risk breaking later engine + // mutation of that same object (e.g. SynthesisAffixProcessRule's + // outWord.SyntacticFeatureStruct.PriorityUnion(...) on a different candidate that happens to share + // the reference) -- exactly the kind of behavior change this probe must never cause. So this is a + // deliberately weak, always-safe hash over the top-level feature keys only; FingerprintEquals above + // always does the real FeatureStruct.ValueEquals, which needs no freeze. + private static int SyntacticFeatureStructWeakHash(FeatureModel.FeatureStruct fs) + { + int acc = 0; + foreach (FeatureModel.Feature f in fs.Features) + acc ^= f.GetHashCode(); + return acc; + } + + private static int UnorderedSetHash(IEnumerable items) + { + int acc = 0; + foreach (T item in items) + acc ^= item?.GetHashCode() ?? 0; + return acc; + } + + private static int UnorderedDictHash( + IReadOnlyDictionary dict, + Func valueHash + ) + { + int acc = 0; + foreach (KeyValuePair kvp in dict) + acc ^= ((kvp.Key?.GetHashCode() ?? 0) * 397) ^ valueHash(kvp.Value); + return acc; + } + + private static bool DictEquals( + IReadOnlyDictionary a, + IReadOnlyDictionary b, + Func valueEquals + ) + { + if (a.Count != b.Count) + return false; + foreach (KeyValuePair kvp in a) + { + if (!b.TryGetValue(kvp.Key, out TValue otherValue) || !valueEquals(kvp.Value, otherValue)) + return false; + } + return true; + } + + private readonly struct FoldStepKey : IEquatable + { + private readonly Word _word; + private readonly IMorphologicalRule _rule; + private readonly int _hash; + + public FoldStepKey(Word word, IMorphologicalRule rule) + { + _word = word; + _rule = rule; + _hash = (FingerprintHash(word) * 397) ^ (rule?.GetHashCode() ?? 0); + } + + public bool Equals(FoldStepKey other) => _rule == other._rule && FingerprintEquals(_word, other._word); + + public override bool Equals(object obj) => obj is FoldStepKey k && Equals(k); + + public override int GetHashCode() => _hash; + } + + // Same fingerprint as FoldStepKey, minus the rule component: N1's dedupe census keys on the + // alternative alone, since it runs at fold entry, before any rule has been chosen/applied. + private readonly struct AlternativeKey : IEquatable + { + private readonly Word _word; + private readonly int _hash; + + public AlternativeKey(Word word) + { + _word = word; + _hash = FingerprintHash(word); + } + + public bool Equals(AlternativeKey other) => FingerprintEquals(_word, other._word); + + public override bool Equals(object obj) => obj is AlternativeKey k && Equals(k); + + public override int GetHashCode() => _hash; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs index a15a6de5..b784b0cb 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using SIL.Extensions; using SIL.Machine.Annotations; @@ -93,7 +94,25 @@ public IEnumerable Apply(Word input) private IEnumerable ApplyMorphologicalRules(Word input) { - foreach (Word mruleOutWord in _mrulesRule.Apply(input)) + // Materialized only when the probe is on: the P1a wall-time split needs a bracketed call to + // time, and this is the entry point to the whole morphological-rule-cascade subsystem for this + // node (the cascade's own internal recursion happens inside _mrulesRule.Apply, so this one call + // captures all of it with no double counting against the template-battery timer below, which + // brackets a disjoint call). Instrumentation only -- the same results are yielded either way. + IEnumerable mruleOutWords; + if (SynthesisProbe.Enabled) + { + long start = Stopwatch.GetTimestamp(); + var materialized = _mrulesRule.Apply(input).ToList(); + SynthesisProbe.AddSynCascadeTicks(Stopwatch.GetTimestamp() - start); + mruleOutWords = materialized; + } + else + { + mruleOutWords = _mrulesRule.Apply(input); + } + + foreach (Word mruleOutWord in mruleOutWords) { if (mruleOutWord.IsLastAppliedRuleFinal ?? false) { @@ -109,7 +128,20 @@ private IEnumerable ApplyMorphologicalRules(Word input) private IEnumerable ApplyTemplates(Word input) { - foreach (Word tempOutWord in _templatesRule.Apply(input)) + IEnumerable templateOutWords; + if (SynthesisProbe.Enabled) + { + long start = Stopwatch.GetTimestamp(); + var materialized = _templatesRule.Apply(input).ToList(); + SynthesisProbe.AddSynBatteryTicks(Stopwatch.GetTimestamp() - start); + templateOutWords = materialized; + } + else + { + templateOutWords = _templatesRule.Apply(input); + } + + foreach (Word tempOutWord in templateOutWords) { switch (_stratum.MorphologicalRuleOrder) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index 51c3c53c..4ca2c0c2 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -404,6 +404,31 @@ internal int GetApplicationCount(IMorphologicalRule mrule) return numApplies; } + /// + /// The full application-count table backing . Read-only exposure + /// for 's P1c fold-step fingerprint (see + /// docs/hermitcrab-synthesis-fold-probes.md section 3) -- mirrors how + /// already exposes the analysis-side equivalent. + /// + internal IReadOnlyDictionary AppliedRuleCounts => _mrulesApplied; + + /// + /// The disjunctive-allomorph-application table, keyed by morph id. Read-only exposure for + /// 's P1c fingerprint: reads this + /// per morph, so two words that differ here can validly produce different outcomes even with + /// everything else equal, and a fingerprint that omitted it would be exactly the kind of + /// incomplete key Word.ValueEquals already is (see the class remarks at the top of this file's + /// callers). + /// + internal IReadOnlyDictionary> DisjunctiveAllomorphIndices => _disjunctiveAllomorphIndices; + + /// + /// The trail index a synthesis step reads via : how far + /// through _mruleApps this word has progressed. Exposed as "pending-trail position" for + /// 's P1c fingerprint. + /// + internal int PendingTrailPosition => _mruleAppIndex; + internal Word CurrentNonHead { get diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/RuleLengthClassifierTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/RuleLengthClassifierTests.cs new file mode 100644 index 00000000..dd244cad --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/RuleLengthClassifierTests.cs @@ -0,0 +1,263 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Static classification of what un-applying a morphological rule does to word length +/// (hermitcrab-forest-memo-plan.md Stage 1). The direction is the thing these tests pin down: a rule +/// that INSERTS on synthesis REMOVES on analysis, and a rule that DELETES on synthesis RESTORES on +/// analysis. Only the first kind can be dropped from . +/// +public class RuleLengthClassifierTests : HermitCrabTestBase +{ + private static FeatureStruct Any => FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private static AffixProcessRule Rule(string name, params AffixProcessAllomorph[] allomorphs) + { + var rule = new AffixProcessRule { Name = name, Gloss = name }; + foreach (AffixProcessAllomorph allomorph in allomorphs) + rule.Allomorphs.Add(allomorph); + return rule; + } + + [Test] + public void OrdinarySuffix_Shrinks() + { + AffixProcessRule rule = Rule( + "s_suffix", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Shrinking)); + } + + [Test] + public void ZeroMorpheme_DoesNotShrink() + { + // The correspondent's own N -> V -> N cycle case: nothing is inserted, so nothing is removed on + // un-application and the shape length measure gives no progress. Must stay in the key. + AffixProcessRule rule = Rule( + "zero", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.NonShrinking)); + } + + [Test] + public void Infix_Shrinks() + { + // Material inserted BETWEEN two copied parts is still material inserted. This is the case most + // likely to be got backwards, so it is asserted explicitly. + AffixProcessRule rule = Rule( + "infix", + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).Value, + Pattern.New("2").Annotation(Any).OneOrMore.Value, + }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s"), new CopyFromInput("2") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Shrinking)); + } + + [Test] + public void Truncation_DoesNotShrink() + { + // An Lhs part with no Rhs copy is deleted on synthesis, so AnalysisMorphologicalTransform + // untruncates it on analysis -- the un-applied word GROWS even though the rule also inserts. + AffixProcessRule rule = Rule( + "truncating", + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).Value, + }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.NonShrinking)); + } + + [Test] + public void Reduplication_IsUnknown() + { + AffixProcessRule rule = Rule( + "redup", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new CopyFromInput("1") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Unknown)); + } + + [Test] + public void Simulfix_DoesNotShrink() + { + // ModifyFromInput captures its part (so nothing is untruncated) but inserts nothing, so the + // un-applied word is exactly as long as the input: length-preserving, and therefore retained. + var voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + AffixProcessRule rule = Rule( + "simulfix", + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).Value, + }, + Rhs = { new CopyFromInput("1"), new ModifyFromInput("2", voiced) }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.NonShrinking)); + } + + [Test] + public void SimulfixPlusInsertion_Shrinks() + { + var voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + AffixProcessRule rule = Rule( + "simulfix_z", + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).Value, + }, + Rhs = { new CopyFromInput("1"), new ModifyFromInput("2", voiced), new InsertSegments(Table3, "z") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Shrinking)); + } + + [Test] + public void BoundaryOnlyInsertion_DoesNotShrink() + { + // "+" is a boundary, not a segment. InsertSegments.GenerateAnalysisLhs deliberately omits boundary + // nodes from the analysis pattern, so the classifier does not count them as removable material. + // Conservative, and therefore safe. + AffixProcessRule rule = Rule( + "boundary_only", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.NonShrinking)); + } + + [Test] + public void WeakestAllomorphGoverns() + { + // One allomorph inserts, the other is a zero. The rule as a whole cannot be relied on to shorten. + AffixProcessRule rule = Rule( + "mixed", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + }, + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.NonShrinking)); + } + + [Test] + public void RealizationalRule_IsClassifiedLikeAnAffixRule() + { + var rule = new RealizationalAffixProcessRule { Name = "real", Gloss = "REAL" }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Shrinking)); + } + + [Test] + public void CompoundingRule_IsUnknown() + { + var rule = new CompoundingRule { Name = "compound" }; + rule.Subrules.Add( + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(Any).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new CopyFromInput("nonHead") }, + } + ); + + Assert.That(RuleLengthClassifier.Classify(rule), Is.EqualTo(UnapplicationLengthEffect.Unknown)); + } + + [Test] + public void RetainInKeyMap_DropsOnlyShrinkingRules() + { + AffixProcessRule suffix = Rule( + "s_suffix", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + } + ); + AffixProcessRule zero = Rule( + "zero", + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(suffix); + Morphophonemic.MorphologicalRules.Add(zero); + + IReadOnlyDictionary map = RuleLengthClassifier.BuildRetainInKeyMap(Language); + + Assert.That(map[suffix], Is.False, "a strictly shrinking rule need not be counted in the key"); + Assert.That(map[zero], Is.True, "a zero morpheme can cycle and must stay in the key"); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldProbe.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldProbe.cs new file mode 100644 index 00000000..48f6d20a --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldProbe.cs @@ -0,0 +1,529 @@ +#nullable disable +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// P1 harness for docs/hermitcrab-synthesis-fold-probes.md section 3: one pass over +/// 's synthesis loop (via ) that emits all three P1 +/// measurements per word -- the wall-time split (P1a), the die-point histogram (P1b), and the +/// fold-step fingerprint ratio (P1c) -- plus the determinism check P1c depends on. +/// +/// Always runs the memoized path (maxDegreeOfParallelism: 1): that is the only sequential +/// cascade, and P1c's premise is entirely about what the memo already identifies as shared. +/// +/// +/// [Explicit] and env-var driven for the real-corpus mode, matching : +/// this repo never commits real grammars or word lists, so the test embeds no grammar content and +/// writes only TestContext lines. The conformance-fixture mode needs no env vars -- the 33 fixtures +/// under conformance/ are synthetic and committed, so their ids and per-word counts are safe to print. +/// Real-corpus output stays to counts, ratios, timings, and die-point categories only -- never rule +/// names, stratum names, morpheme glosses, or lexical entries. +/// +/// +/// # breadth: all 33 conformance fixtures, no env vars needed +/// dotnet test --filter "FullyQualifiedName~SynthesisFoldProbe.Probe_ConformanceFixtures" +/// +/// # depth: a real corpus +/// $env:HC_MEMO_GRAMMAR = "...\sena-hc.xml" +/// $env:HC_MEMO_WORDS = "...\sena-words.txt" # optional if HC_PROBE_WORDS is set +/// $env:HC_MEMO_MAX_WORDS = "60" # optional, default 60 +/// $env:HC_PROBE_WORDS = "atawirambo,cinacemerwa" # optional explicit override, comma-separated +/// dotnet test --filter "FullyQualifiedName~SynthesisFoldProbe.Probe_RealCorpus" +/// +/// +[TestFixture] +[Explicit("Manual instrumentation run; not part of CI. See docs/hermitcrab-synthesis-fold-probes.md.")] +public class SynthesisFoldProbe +{ + private static readonly SynthesisDiePoint[] AllDiePoints = (SynthesisDiePoint[]) + Enum.GetValues(typeof(SynthesisDiePoint)); + + [Test] + public void Probe_ConformanceFixtures() + { + string fixturesRoot = Environment.GetEnvironmentVariable("HC_PROBE_FIXTURES_ROOT"); + if (string.IsNullOrEmpty(fixturesRoot)) + fixturesRoot = Path.Combine(RepositoryRoot(), "conformance"); + + List fixtures = Fixture.DiscoverAll(fixturesRoot); + Assert.That(fixtures, Is.Not.Empty, $"no fixtures discovered under {fixturesRoot}"); + + SynthesisProbe.Enabled = true; + long grandDeterminismViolations = 0; + var fixtureRatios = new List(); + try + { + foreach (Fixture fixture in fixtures) + { + Language language; + try + { + language = XmlLanguageLoader.Load(fixture.GrammarPath); + } + catch (Exception e) + { + TestContext.Out.WriteLine($"[{fixture.Id}] grammar failed to load: {e.GetType().Name} -- skipped"); + continue; + } + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + SynthesisProbe.ResetFoldSteps(); + + var rows = new List(); + foreach (WordEntry entry in fixture.Words.Words) + { + WordProbeResult row = ProbeWord(morpher, entry.Word); + if (row != null) + rows.Add(row); + } + + PrintFixtureSummary(fixture.Id, rows); + long applications = SynthesisProbe.TotalApplications; + long distinct = SynthesisProbe.DistinctFoldSteps; + double ratio = distinct > 0 ? applications / (double)distinct : 0; + + // Corrected share for fold-step sharing (docs/hermitcrab-synthesis-fold-probes.md section + // 6.4): synForward is explicitly NET of the cascade/battery brackets (Morpher.cs:424), but + // every application P1c counts happens inside SynthesisAffixProcessRule.Apply / + // SynthesisRealizationalAffixProcessRule.Apply, which run INSIDE the synCascade/synBattery + // brackets (template slot rules compile to those same classes via RuleBatch). So the + // shareable work lives in synCascade + synBattery + synForward, not synForward alone -- + // dividing by synForward alone is the error section 6.4 found and corrected. + double fixtureWall = rows.Sum(r => r.WallMs); + double fixtureSynCascade = rows.Sum(r => r.SynCascadeMs); + double fixtureSynBattery = rows.Sum(r => r.SynBatteryMs); + double fixtureSynForward = rows.Sum(r => r.SynForwardMs); + double fixtureSynExpand = rows.Sum(r => r.SynExpandMs); + double synTotalShare = + fixtureWall > 0 + ? (fixtureSynCascade + fixtureSynBattery + fixtureSynForward) / fixtureWall + : 0; + double synExpandShare = fixtureWall > 0 ? fixtureSynExpand / fixtureWall : 0; + // share x (1 - 1/ratio), as a percentage (docs section 6.4's "corrected ceilings" formula). + double foldStepCeiling = ratio > 0 ? synTotalShare * (1 - 1 / ratio) * 100 : 0; + + // N1 fold-entry census, per fixture (SynthesisProbe.ResetFoldSteps above also resets the + // fold-entry counters, so these are this fixture's own totals, not cumulative across + // fixtures -- see section 6.4's "one honest gap" / dedupe census). + long altTotal = SynthesisProbe.TotalAlternatives; + long altDistinct = SynthesisProbe.DistinctAlternatives; + double altDistinctPct = altTotal > 0 ? altDistinct / (double)altTotal * 100 : 0; + long dupeSame = SynthesisProbe.DupeSameAnalysisWord; + long dupeDifferent = SynthesisProbe.DupeDifferentAnalysisWord; + long totalDupes = dupeSame + dupeDifferent; + double dupeSameWordPct = totalDupes > 0 ? dupeSame / (double)totalDupes * 100 : 0; + + fixtureRatios.Add( + new FixtureSummaryRow + { + Id = fixture.Id, + Ratio = ratio, + Applications = applications, + Distinct = distinct, + SynTotalShare = synTotalShare, + SynExpandShare = synExpandShare, + FoldStepCeiling = foldStepCeiling, + AltTotal = altTotal, + AltDistinct = altDistinct, + AltDistinctPct = altDistinctPct, + DupeSameWordPct = dupeSameWordPct, + WallMs = fixtureWall, + Words = rows.Count, + } + ); + grandDeterminismViolations += SynthesisProbe.DeterminismViolations; + } + + fixtureRatios.Sort((a, b) => b.Ratio.CompareTo(a.Ratio)); + + TestContext.Out.WriteLine(); + TestContext.Out.WriteLine( + "=== P1c ratio by fixture (not pooled -- fixtures vary wildly in size), sorted by ratio " + + "descending; synTotalShare = (synCascade+synBattery+synForward)/wall (the corrected " + + "share, docs section 6.4); foldStepCeiling = synTotalShare x (1 - 1/ratio); " + + "reliable = wallMs >= 50 ===" + ); + TestContext.Out.WriteLine( + " id\tapplications\tdistinct\tratio\tsynTotalShare\tsynExpandShare\tfoldStepCeiling\t" + + "altTotal\taltDistinct\taltDistinctPct\tdupeSameWordPct\twallMs\twords\treliable" + ); + foreach (FixtureSummaryRow r in fixtureRatios) + { + TestContext.Out.WriteLine( + $" {r.Id}\t{r.Applications}\t{r.Distinct}\t{r.Ratio:F2}x\t" + + $"{r.SynTotalShare * 100:F1}%\t{r.SynExpandShare * 100:F1}%\t{r.FoldStepCeiling:F2}%\t" + + $"{r.AltTotal}\t{r.AltDistinct}\t{r.AltDistinctPct:F2}%\t{r.DupeSameWordPct:F1}%\t" + + $"{r.WallMs:F2}\t{r.Words}\t{(r.WallMs >= 50 ? "yes" : "no")}" + ); + } + TestContext.Out.WriteLine(); + TestContext.Out.WriteLine( + $"=== DETERMINISM VIOLATIONS across all fixtures: {grandDeterminismViolations} " + + $"{(grandDeterminismViolations > 0 ? "(fingerprint is INCOMPLETE -- see class remarks)" : "(none observed)")} ===" + ); + } + finally + { + SynthesisProbe.Enabled = false; + SynthesisProbe.ResetAll(); + } + } + + [Test] + public void Probe_RealCorpus() + { + (Language language, List words) = LoadRealCorpus(); + + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + SynthesisProbe.Enabled = true; + SynthesisProbe.ResetFoldSteps(); + try + { + var rows = new List(); + foreach (string word in words) + { + WordProbeResult row = ProbeWord(morpher, word); + if (row != null) + rows.Add(row); + } + + PrintFixtureSummary("real-corpus", rows); + long applications = SynthesisProbe.TotalApplications; + long distinct = SynthesisProbe.DistinctFoldSteps; + double ratio = distinct > 0 ? applications / (double)distinct : 0; + long violations = SynthesisProbe.DeterminismViolations; + TestContext.Out.WriteLine(); + TestContext.Out.WriteLine( + $"=== P1c: applications={applications}, distinct={distinct}, ratio={ratio:F2}x ===" + ); + TestContext.Out.WriteLine( + $"=== DETERMINISM VIOLATIONS: {violations} " + + $"{(violations > 0 ? "(fingerprint is INCOMPLETE -- see SynthesisProbe class remarks)" : "(none observed)")} ===" + ); + } + finally + { + SynthesisProbe.Enabled = false; + SynthesisProbe.ResetAll(); + } + } + + /// + /// One row of the per-fixture summary table printed at the end of . + /// See docs/hermitcrab-synthesis-fold-probes.md section 6.4 for the corrected share formula this + /// replaces the old (wrong) forwardShare/value columns with. + /// + private sealed class FixtureSummaryRow + { + public string Id; + public double Ratio; + public long Applications; + public long Distinct; + public double SynTotalShare; + public double SynExpandShare; + public double FoldStepCeiling; + public long AltTotal; + public long AltDistinct; + public double AltDistinctPct; + public double DupeSameWordPct; + public double WallMs; + public int Words; + } + + private sealed class WordProbeResult + { + public string Word; + public int ParseCount; + public double WallMs; + public double LexicalLookupMs; + public double SynCascadeMs; + public double SynBatteryMs; + public double SynForwardMs; + public double SynExpandMs; + public double AnTotalMs; + public double AnCascadeMs; + public double AnBatteryMs; + public double AnPhonoMs; + public double UnaccountedMs; + public long[] DieCounts; + public long ApplicationsThisWord; + public long NewDistinctThisWord; + + // N1 dedupe census (docs/hermitcrab-synthesis-fold-probes.md section 6.4's "one honest gap"): + // alternatives entering the fold this word, how many were new distinct fingerprints, and of the + // duplicates, how many trace their first occurrence to the same analysis word vs. a different one. + public long AlternativesThisWord; + public long NewDistinctAlternativesThisWord; + public long DupeSameThisWord; + public long DupeDifferentThisWord; + + // Forward-synthesis share of wall time -- the multiplier the P1c sharing ratio needs to turn into + // an actual expected win (P1c ratio x this share; see the "value" column in PrintFixtureSummary). + // Sharing that never reaches forward synthesis (the cascade/battery/lookup buckets, or analysis + // time) cannot be realized by folding forward-synthesis steps. + public double SynForwardShare => WallMs > 0 ? SynForwardMs / WallMs : 0; + } + + /// + /// Runs one word through the instrumented sequential path and returns its P1a/P1b/P1c deltas. + /// P1a/P1b timers and counters are reset immediately before the word (so the result is a per-word + /// delta); the P1c fold-step table is NOT reset here -- it accumulates across a whole fixture/corpus + /// so the distinct-pair count is a true count over the combined stream, and this method instead + /// snapshots / + /// before and after to report this word's own contribution. + /// + private static WordProbeResult ProbeWord(Morpher morpher, string word) + { + SynthesisProbe.ResetWallTime(); + SynthesisProbe.ResetDiePoints(); + long applicationsBefore = SynthesisProbe.TotalApplications; + long distinctBefore = SynthesisProbe.DistinctFoldSteps; + long alternativesBefore = SynthesisProbe.TotalAlternatives; + long distinctAltBefore = SynthesisProbe.DistinctAlternatives; + long dupeSameBefore = SynthesisProbe.DupeSameAnalysisWord; + long dupeDifferentBefore = SynthesisProbe.DupeDifferentAnalysisWord; + + int parseCount; + var wall = Stopwatch.StartNew(); + try + { + parseCount = morpher.ParseWord(word).Count(); + } + catch (InvalidShapeException) + { + // As Morpher.AnalyzeWord/MemoCorpusVerification do: a word list can contain strings the + // character table does not cover. Both memo-on and memo-off reject these identically and no + // synthesis is attempted, so there is nothing for this probe to measure. + return null; + } + catch (Exception e) + { + // A handful of edge-case fixtures have ExpectCrash: true -- their ground truth IS a thrown + // exception (e.g. an epenthesis rule hitting the infinite-loop guard), not a signature. This + // probe is not the self-check for that contract (Fixture's own self-check already covers it), + // so it just notes the crash and skips measurement for this word rather than aborting the + // whole fixture/run. The exception type name is a .NET framework/engine identifier, not + // grammar content, so it is safe to print. + TestContext.Out.WriteLine($" {word}\tCRASHED: {e.GetType().Name}"); + return null; + } + wall.Stop(); + + var dieCounts = new long[AllDiePoints.Length]; + for (int i = 0; i < AllDiePoints.Length; i++) + dieCounts[i] = SynthesisProbe.GetDieCount(AllDiePoints[i]); + + double wallMs = wall.Elapsed.TotalMilliseconds; + double lookupMs = TicksToMs(SynthesisProbe.LexicalLookupTicks); + double synCascadeMs = TicksToMs(SynthesisProbe.SynCascadeTicks); + double synBatteryMs = TicksToMs(SynthesisProbe.SynBatteryTicks); + double synForwardMs = TicksToMs(SynthesisProbe.SynForwardTicks); + double synExpandMs = TicksToMs(SynthesisProbe.SynExpandTicks); + double anTotalMs = TicksToMs(SynthesisProbe.AnTotalTicks); + double anCascadeMs = TicksToMs(SynthesisProbe.AnCascadeTicks); + double anBatteryMs = TicksToMs(SynthesisProbe.AnBatteryTicks); + double anPhonoMs = TicksToMs(SynthesisProbe.AnPhonoTicks); + + // Top-level buckets are disjoint by construction (see SynthesisProbe's wall-time-split remarks): + // AnTotalMs is the whole analysis phase (a nested/inclusive total that already contains + // AnCascade/AnBattery/AnPhono), and the syn*/lookup/synExpand buckets are disjoint slices of the + // synthesis phase (SynForwardMs is already net of SynCascade/SynBattery, see + // Morpher.SynthesizeSequential; SynExpandMs is N1's new bracket around ExpandAlternatives() itself, + // also disjoint from all of those -- it wraps the call, not anything inside SynCascade/SynBattery/ + // SynForward). So unaccounted = wall - analysis phase - synthesis phase, i.e. ParseWord's own + // scaffolding (shape segmentation, Word construction/Freeze, AccumulateMemoDiagnostics, guessRoot) + // plus any region this probe does not yet bracket. + double unaccountedMs = wallMs - anTotalMs - lookupMs - synCascadeMs - synBatteryMs - synForwardMs - synExpandMs; + + return new WordProbeResult + { + Word = word, + ParseCount = parseCount, + WallMs = wallMs, + LexicalLookupMs = lookupMs, + SynCascadeMs = synCascadeMs, + SynBatteryMs = synBatteryMs, + SynForwardMs = synForwardMs, + SynExpandMs = synExpandMs, + AnTotalMs = anTotalMs, + AnCascadeMs = anCascadeMs, + AnBatteryMs = anBatteryMs, + AnPhonoMs = anPhonoMs, + UnaccountedMs = unaccountedMs, + DieCounts = dieCounts, + ApplicationsThisWord = SynthesisProbe.TotalApplications - applicationsBefore, + NewDistinctThisWord = SynthesisProbe.DistinctFoldSteps - distinctBefore, + AlternativesThisWord = SynthesisProbe.TotalAlternatives - alternativesBefore, + NewDistinctAlternativesThisWord = SynthesisProbe.DistinctAlternatives - distinctAltBefore, + DupeSameThisWord = SynthesisProbe.DupeSameAnalysisWord - dupeSameBefore, + DupeDifferentThisWord = SynthesisProbe.DupeDifferentAnalysisWord - dupeDifferentBefore, + }; + } + + private static double TicksToMs(long stopwatchTicks) => stopwatchTicks * 1000.0 / Stopwatch.Frequency; + + private static void PrintFixtureSummary(string fixtureId, List rows) + { + TestContext.Out.WriteLine(); + TestContext.Out.WriteLine($"--- {fixtureId} ({rows.Count} word(s) measured) ---"); + // Nesting scheme (P1a follow-up, docs/hermitcrab-synthesis-fold-probes.md section 3): anTotal is a + // NESTED/INCLUSIVE total for the whole analysis phase (it contains anCascade+anBattery+anPhono, so + // those three do not add on top of it). Every other column here -- lookup, synCascade, synBattery, + // synForward, anTotal, unaccounted -- is an EXCLUSIVE slice of wall time; those six sum to wall + // exactly (unaccounted is defined as the remainder). "apps+"/"newDistinct+" are P1c counters, not + // wall-time buckets. + foreach (WordProbeResult r in rows) + { + double wordRatio = r.NewDistinctAlternativesThisWord > 0 + ? r.AlternativesThisWord / (double)r.NewDistinctAlternativesThisWord + : 0; + TestContext.Out.WriteLine( + $" {r.Word}\tparses={r.ParseCount}\twall={r.WallMs:F2}ms\t" + + $"lookup={r.LexicalLookupMs:F2}\tsynCascade={r.SynCascadeMs:F2}\t" + + $"synBattery={r.SynBatteryMs:F2}\tsynForward={r.SynForwardMs:F2}\tsynExpand={r.SynExpandMs:F2}\t" + + $"anTotal={r.AnTotalMs:F2} [anCascade={r.AnCascadeMs:F2} anBattery={r.AnBatteryMs:F2} anPhono={r.AnPhonoMs:F2}]\t" + + $"unaccounted={r.UnaccountedMs:F2}\t" + + $"apps+={r.ApplicationsThisWord}\tnewDistinct+={r.NewDistinctThisWord}\t" + + $"[N1] alternatives+={r.AlternativesThisWord}\tdistinctAlternatives+={r.NewDistinctAlternativesThisWord}" + + $"\tdistinct/total={Pct(r.NewDistinctAlternativesThisWord, r.AlternativesThisWord)} (ratio={wordRatio:F2}x)" + + $"\tdupeSame={r.DupeSameThisWord}\tdupeDifferent={r.DupeDifferentThisWord}" + ); + } + + if (rows.Count == 0) + { + TestContext.Out.WriteLine(" (no words measured)"); + return; + } + + double sumWall = rows.Sum(r => r.WallMs); + double sumLookup = rows.Sum(r => r.LexicalLookupMs); + double sumSynCascade = rows.Sum(r => r.SynCascadeMs); + double sumSynBattery = rows.Sum(r => r.SynBatteryMs); + double sumSynForward = rows.Sum(r => r.SynForwardMs); + double sumSynExpand = rows.Sum(r => r.SynExpandMs); + double sumAnTotal = rows.Sum(r => r.AnTotalMs); + double sumAnCascade = rows.Sum(r => r.AnCascadeMs); + double sumAnBattery = rows.Sum(r => r.AnBatteryMs); + double sumAnPhono = rows.Sum(r => r.AnPhonoMs); + double sumUnaccounted = rows.Sum(r => r.UnaccountedMs); + TestContext.Out.WriteLine( + $" [P1a totals -- exclusive slices, sum to wall] wall={sumWall:F2}ms " + + $"lookup={sumLookup:F2}ms ({Pct(sumLookup, sumWall)}) " + + $"synCascade={sumSynCascade:F2}ms ({Pct(sumSynCascade, sumWall)}) " + + $"synBattery={sumSynBattery:F2}ms ({Pct(sumSynBattery, sumWall)}) " + + $"synForward={sumSynForward:F2}ms ({Pct(sumSynForward, sumWall)}) " + + $"synExpand={sumSynExpand:F2}ms ({Pct(sumSynExpand, sumWall)}) " + + $"anTotal={sumAnTotal:F2}ms ({Pct(sumAnTotal, sumWall)}) " + + $"unaccounted={sumUnaccounted:F2}ms ({Pct(sumUnaccounted, sumWall)})" + ); + TestContext.Out.WriteLine( + $" [anTotal breakdown -- nested inside anTotal, not on top of it] " + + $"anCascade={sumAnCascade:F2}ms ({Pct(sumAnCascade, sumAnTotal)} of anTotal) " + + $"anBattery={sumAnBattery:F2}ms ({Pct(sumAnBattery, sumAnTotal)} of anTotal) " + + $"anPhono={sumAnPhono:F2}ms ({Pct(sumAnPhono, sumAnTotal)} of anTotal) " + + $"anOther={sumAnTotal - sumAnCascade - sumAnBattery - sumAnPhono:F2}ms " + + $"({Pct(sumAnTotal - sumAnCascade - sumAnBattery - sumAnPhono, sumAnTotal)} of anTotal)" + ); + + var dieTotals = new long[AllDiePoints.Length]; + foreach (WordProbeResult r in rows) + { + for (int i = 0; i < AllDiePoints.Length; i++) + dieTotals[i] += r.DieCounts[i]; + } + long dieGrandTotal = dieTotals.Sum(); + TestContext.Out.Write(" [P1b die points] "); + if (dieGrandTotal == 0) + { + TestContext.Out.WriteLine("no rejections recorded"); + } + else + { + TestContext.Out.WriteLine( + string.Join( + " ", + AllDiePoints.Select( + (p, i) => $"{p}={dieTotals[i]} ({Pct(dieTotals[i], dieGrandTotal)})" + ) + ) + ); + } + + long applications = SynthesisProbe.TotalApplications; + long distinct = SynthesisProbe.DistinctFoldSteps; + double ratio = distinct > 0 ? applications / (double)distinct : 0; + TestContext.Out.WriteLine( + $" [P1c] applications={applications} distinct={distinct} ratio={ratio:F2}x " + + $"(cumulative for this fixture/corpus so far)" + ); + + // N1 dedupe census, cumulative (docs/hermitcrab-synthesis-fold-probes.md section 6.4's "one honest + // gap"). distinctAlternatives/totalAlternatives is the gate's "distinct/total" ratio; dupeSame vs. + // dupeDifferent is the decisive provenance split -- same-analysis-word duplication is interceptable + // BEFORE ExpandAlternatives' Clone/Unify/Freeze work, cross-analysis-word duplication only AFTER it. + long totalAlternatives = SynthesisProbe.TotalAlternatives; + long distinctAlternatives = SynthesisProbe.DistinctAlternatives; + long dupeSame = SynthesisProbe.DupeSameAnalysisWord; + long dupeDifferent = SynthesisProbe.DupeDifferentAnalysisWord; + double distinctOverTotal = totalAlternatives > 0 ? distinctAlternatives / (double)totalAlternatives : 0; + long totalDupes = dupeSame + dupeDifferent; + TestContext.Out.WriteLine( + $" [N1] totalAlternatives={totalAlternatives} distinctAlternatives={distinctAlternatives} " + + $"distinct/total={distinctOverTotal:F3} ({Pct(distinctAlternatives, totalAlternatives)}) " + + $"dupeSameAnalysisWord={dupeSame} ({Pct(dupeSame, totalDupes)} of dupes) " + + $"dupeDifferentAnalysisWord={dupeDifferent} ({Pct(dupeDifferent, totalDupes)} of dupes) " + + $"(cumulative for this fixture/corpus so far)" + ); + } + + private static string Pct(double part, double whole) => whole > 0 ? $"{part / whole * 100:F1}%" : "n/a"; + + private static string RepositoryRoot() + { + string directory = TestContext.CurrentContext.TestDirectory; + while (directory != null) + { + if (File.Exists(Path.Combine(directory, "conformance", "constructs.txt"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root."); + return string.Empty; + } + + private static (Language, List) LoadRealCorpus() + { + string grammarPath = Environment.GetEnvironmentVariable("HC_MEMO_GRAMMAR"); + if (string.IsNullOrEmpty(grammarPath)) + Assert.Ignore("set HC_MEMO_GRAMMAR (and either HC_PROBE_WORDS or HC_MEMO_WORDS)"); + + Language language = XmlLanguageLoader.Load(grammarPath!); + + string explicitWords = Environment.GetEnvironmentVariable("HC_PROBE_WORDS"); + if (!string.IsNullOrEmpty(explicitWords)) + { + List words = explicitWords! + .Split(',') + .Select(w => w.Trim()) + .Where(w => w.Length > 0) + .ToList(); + return (language, words); + } + + string wordsPath = Environment.GetEnvironmentVariable("HC_MEMO_WORDS"); + if (string.IsNullOrEmpty(wordsPath)) + Assert.Ignore("set HC_MEMO_GRAMMAR (and either HC_PROBE_WORDS or HC_MEMO_WORDS)"); + + int maxWords = int.TryParse(Environment.GetEnvironmentVariable("HC_MEMO_MAX_WORDS"), out int mw) ? mw : 60; + List fileWords = File.ReadAllLines(wordsPath!) + .Select(w => w.Trim()) + .Where(w => w.Length > 0) + .Take(maxWords) + .ToList(); + return (language, fileWords); + } +}