diff --git a/docs/hermitcrab-synthesis-fold-probes.md b/docs/hermitcrab-synthesis-fold-probes.md index 8c193d38..a617d5a4 100644 --- a/docs/hermitcrab-synthesis-fold-probes.md +++ b/docs/hermitcrab-synthesis-fold-probes.md @@ -626,3 +626,52 @@ Commit `1a7d484c` bundles the harness reporting change with the section 8 analys agent that wrote the reporting change stopped with it uncommitted and the work was swept in when the analysis was committed. Two concerns in one commit; flagged here rather than rewritten, since the PR body is the review artifact. +--- + +## 10. The build was made, and it does not pay — because the redundancy is not real + +`feature/synthesis-fold-sharing` (`1048c742`) implements the fold-step memo with a sound key: +`SynthesisStateKey` carries the **ordered remaining trail**, not just its index, plus shape, +syntactic FS, realizational FS, MPR set, root allomorph, disjunctive allomorph indices, applied +counts, `IsPartial`, `IsLastAppliedRuleFinal` and stratum — each field justified against its reader. + +**Parity: 0 divergences across all 33 conformance fixtures.** The implementation is correct. + +**Measured (warm-up discarded, min of 5 interleaved samples per arm):** + +| fixture | predicted ceiling | realised | memo hits | off-arm spread | +| --- | --- | --- | --- | --- | +| deep-optional-affix-nesting | 50.4% (~2x) | **0.96x** (4.5% slower) | **0** | 38.6% | +| suffixing-evidential-adjacency-chain | 52.0% (~2x) | 1.06x (5.4%) | 2,682 | 21.9% | + +**`hits = 0` on the fixture with the largest reliable sample and the highest predicted ceiling.** +With a trail-complete key the memo never fires there at all; the 0.96x is pure key-construction +overhead. On the evidential chain it does fire 2,682 times and returns 5.4% against 21.9% noise — +nothing. + +### The structural finding + +This is the same result arriving a third time, by a third independent route: + +| measurement | key | apparent | sound | +| --- | --- | --- | --- | +| F1 synthesis-input dedupe | order-insensitive | 9,774x | 15–40% | +| N1 fold-entry census | trail position only | 6,476x | not established | +| **P1c fold-step sharing** | **trail position only** | **3.22x / 8.10x** | **hits=0 / 1.06x** | + +**The redundancy in HermitCrab's synthesis is apparent, not real. The trail is what makes each +step distinct, and every measurement that shows large shareable work is measuring a key that omits +it.** Three different boundaries, three spectacular ratios, three collapses under a complete key. + +That closes a family, not just a candidate: packed parse forests, fold-step sharing, and +synthesis-input dedupe all depend on distinct derivations converging on a genuinely identical +state, and in this engine they do not converge. It is the same fact that makes the rules +non-order-invariant, seen from the other side. + +### Recommendation + +**Do not merge the fold-step memo as a performance feature.** It is correct, it is off by default, +and it buys nothing — on the best case it is 4% slower. Keep the branch as the evidence that +closes the family. + +The measurement infrastructure is the durable asset and should merge on its own. diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 6d756369..344a4825 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -30,6 +30,7 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator private long _nogoodHits; private long _templateMemoHits; private long _templateNogoodHits; + private long _synthesisFoldHits; public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0) { @@ -89,6 +90,19 @@ public ITraceManager TraceManager /// public bool MergeEquivalentAnalyses { get; set; } + /// + /// Turns on the synthesis fold-step memo (docs/hermitcrab-synthesis-fold-probes.md), which shares + /// / + /// applications across + /// candidates that reach an equal . Defaults to false: this is + /// a plain A/B toggle so both configurations can be measured in one process, not a change to any + /// default behaviour. Only takes effect in Morpher.SynthesizeSequential + /// ( == 1, untraced) -- same restriction as the analysis-cascade + /// memo, and for the same reason: 's table is a plain + /// , not thread-safe. + /// + public bool UseSynthesisFoldMemo { get; set; } + /// /// Caps the concurrency used within a single parse or generation -- analysis cascade, /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully @@ -125,6 +139,13 @@ internal ParallelOptions CreateParallelOptions(int uncappedDegree = -1) internal long NogoodHits => Interlocked.Read(ref _nogoodHits); internal long TemplateMemoHits => Interlocked.Read(ref _templateMemoHits); internal long TemplateNogoodHits => Interlocked.Read(ref _templateNogoodHits); + internal long SynthesisFoldHits => Interlocked.Read(ref _synthesisFoldHits); + + // Interlocked because one Morpher may be parsing on several threads at once. + private void AccumulateSynthesisFoldDiagnostics(SynthesisFoldScope scope) + { + Interlocked.Add(ref _synthesisFoldHits, scope.Hits); + } // Interlocked because one Morpher may be parsing on several threads at once. private void AccumulateMemoDiagnostics(AnalysisScope scope) @@ -366,6 +387,13 @@ private IEnumerable Synthesize(string word, ConcurrentQueue analyses private IEnumerable SynthesizeSequential(string word, IEnumerable analyses) { var matches = new HashSet(FreezableEqualityComparer.Default); + // One scope for the whole surface-word parse, shared across every analysis word's alternatives + // -- exactly the scope P1c/N1 measured sharing over (docs/hermitcrab-synthesis-fold-probes.md). + // Never while tracing, matching AnalysisScope's own restriction: traces must stay byte-identical + // to the unmemoized engine. This method is only ever reached when MaxDegreeOfParallelism == 1 + // (see Synthesize), so no separate parallelism check is needed here. + SynthesisFoldScope foldScope = + UseSynthesisFoldMemo && !_traceManager.IsTracing ? new SynthesisFoldScope() : null; int alternativeCount = 0; foreach (Word analysisWord in analyses) { @@ -413,6 +441,9 @@ private IEnumerable SynthesizeSequential(string word, IEnumerable an if (MaxAlternatives > 0 && alternativeCount > MaxAlternatives) throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); + if (foldScope != null) + alternative.SynthesisFoldScope = foldScope; + if (!SynthesisProbe.Enabled) { foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) @@ -447,6 +478,8 @@ private IEnumerable SynthesizeSequential(string word, IEnumerable an } } } + if (foldScope != null) + AccumulateSynthesisFoldDiagnostics(foldScope); return matches; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs index 73d079ab..6940c54a 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs @@ -40,12 +40,40 @@ public SynthesisAffixProcessRule(Morpher morpher, AffixProcessRule rule) public IEnumerable Apply(Word input) { + // The trail-position gate stays outside the memo unconditionally, on both the memoized and + // unmemoized paths: it is an O(1) index-plus-reference check (docs/hermitcrab-synthesis-fold-probes.md + // section 3's "~40x free" observation), so a SynthesisStateKey is never worth constructing for + // the ~72-73% of rejections that die here (section 6.1's P1b histogram). if (!input.IsMorphologicalRuleApplicable(_rule)) { SynthesisProbe.RecordDie(SynthesisDiePoint.RuleNotApplicableOrPatternMismatch); return Enumerable.Empty(); } + SynthesisFoldScope foldScope = _morpher.UseSynthesisFoldMemo ? input.SynthesisFoldScope : null; + if (foldScope == null) + return ApplyMatchingAllomorphs(input); + + SynthesisStateKey key = SynthesisStateKey.PinAndKey(input); + if (foldScope.TryGet(key, _rule, out IReadOnlyList stored)) + { + foldScope.Hits++; + var replayed = new List(stored.Count); + foreach (Word storedOutput in stored) + replayed.Add(storedOutput.ReanchorSynthesisStep(input, trailConsuming: true)); + return replayed; + } + + var computed = ApplyMatchingAllomorphs(input).ToList(); + foldScope.Store(key, _rule, computed); + return computed; + } + + // Everything past the trail-position gate: this is the expensive part of the step (per-allomorph + // MPR checks, pattern matching, unification), and everything it reads is covered by + // SynthesisStateKey -- see that class's doc comment for the field-by-field audit. + private IEnumerable ApplyMatchingAllomorphs(Word input) + { if (input.GetApplicationCount(_rule) >= _rule.MaxApplicationCount) { if (_morpher.TraceManager.IsTracing) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs index 5e72e2e6..bacaad36 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs @@ -43,6 +43,31 @@ public IEnumerable Apply(Word input) if (!_morpher.RuleSelector(_rule)) return Enumerable.Empty(); + SynthesisFoldScope foldScope = _morpher.UseSynthesisFoldMemo ? input.SynthesisFoldScope : null; + if (foldScope == null) + return ApplyUncached(input); + + SynthesisStateKey key = SynthesisStateKey.PinAndKey(input); + if (foldScope.TryGet(key, _rule, out IReadOnlyList stored)) + { + foldScope.Hits++; + var replayed = new List(stored.Count); + foreach (Word storedOutput in stored) + // Realizational rules are trail-exempt (no IsMorphologicalRuleApplicable gate), so a + // successful application never advances PendingTrailPosition. + replayed.Add(storedOutput.ReanchorSynthesisStep(input, trailConsuming: false)); + return replayed; + } + + var computed = ApplyUncached(input).ToList(); + foldScope.Store(key, _rule, computed); + return computed; + } + + // Everything SynthesisStateKey is audited against: the "at most once" and subsumption/blocking + // gates, per-allomorph MPR checks, pattern matching, unification. + private IEnumerable ApplyUncached(Word input) + { // RealizationalRule has no multipleApplication attribute, so it applies at most once per // word; otherwise a rule cascade that retries a matching rule against its own output would // never terminate. diff --git a/src/SIL.Machine.Morphology.HermitCrab/SynthesisFoldScope.cs b/src/SIL.Machine.Morphology.HermitCrab/SynthesisFoldScope.cs new file mode 100644 index 00000000..5d9c89ec --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/SynthesisFoldScope.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// Carrier for the synthesis fold-step memo (docs/hermitcrab-synthesis-fold-probes.md). Same shape as + /// : one instance per + /// call, threaded through , not installed while tracing or when + /// running with more than one degree of parallelism (plain , not + /// thread-safe). + /// + /// A memo entry is keyed on (, applied rule) and holds a SET of output + /// Words -- not one value -- because several allomorphs of one rule can legitimately all pattern-match + /// one input before a disjunctive break (see the doc comment on SynthesisAffixProcessRule.Apply's + /// call to SynthesisProbe.RecordApplications, which records fold steps the same way for the same + /// reason). Stored Words are never handed to a caller directly: every read goes through + /// , which re-parents the stored result onto the querying + /// candidate's own trail/non-head identity. + /// + /// + /// Only successful steps are stored, mirroring : a + /// trail-position mismatch (IsMorphologicalRuleApplicable false) is an O(1) index-plus-reference + /// check the plan doc itself characterizes as "not 95% of anything" at ~29ns, so memoizing it would + /// spend a scarce entry slot buying almost nothing. Empty results *past* that cheap gate (a rule that + /// was trail-eligible but whose allomorphs all failed to unify/pattern-match) ARE stored, because + /// reaching that verdict is exactly the expensive work (unification, pattern matching) this memo exists + /// to share. + /// + /// + internal sealed class SynthesisFoldScope + { + // Same cap philosophy as AnalysisScope.MaxMemoEntries: a coarse backstop, not a figure derived from + // measured memory. Correctness never depends on it -- past the cap, new fold steps are simply + // computed and returned unmemoized, same as when the scope itself is null. + private const int MaxMemoEntries = 100_000; + + private readonly Dictionary> _memo = + new Dictionary>(); + + /// Per-parse hit count, folded into the owning Morpher when the parse ends. + public int Hits { get; set; } + + // Diagnostic totals for the A/B harness: distinguishes "the memo never hits" (the idea is dead) + // from "the memo hits but the key costs more than the step it saves" (the key is the problem). + // Free-running; a harness reads the delta. + internal static long DiagHits; + internal static long DiagStores; + internal static long DiagLookups; + + public bool TryGet(SynthesisStateKey key, IMorphologicalRule rule, out IReadOnlyList outputs) + { + DiagLookups++; + bool hit = _memo.TryGetValue(new SynthesisFoldStepKey(key, rule), out outputs); + if (hit) + DiagHits++; + return hit; + } + + public void Store(SynthesisStateKey key, IMorphologicalRule rule, IReadOnlyList outputs) + { + if (_memo.Count >= MaxMemoEntries) + return; + DiagStores++; + _memo[new SynthesisFoldStepKey(key, rule)] = outputs; + } + + private readonly struct SynthesisFoldStepKey : IEquatable + { + private readonly SynthesisStateKey _state; + private readonly IMorphologicalRule _rule; + private readonly int _hash; + + public SynthesisFoldStepKey(SynthesisStateKey state, IMorphologicalRule rule) + { + _state = state; + _rule = rule; + _hash = (state.GetHashCode() * 397) ^ (rule?.GetHashCode() ?? 0); + } + + public bool Equals(SynthesisFoldStepKey other) => _rule == other._rule && _state.Equals(other._state); + + public override bool Equals(object obj) => obj is SynthesisFoldStepKey k && Equals(k); + + public override int GetHashCode() => _hash; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/SynthesisStateKey.cs b/src/SIL.Machine.Morphology.HermitCrab/SynthesisStateKey.cs new file mode 100644 index 00000000..56d04993 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/SynthesisStateKey.cs @@ -0,0 +1,272 @@ +using System; +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// Identity of a synthesis fold-step input, for + /// (docs/hermitcrab-synthesis-fold-probes.md). Two Words with an equal key must make an identical + /// decision -- same output set -- for the same rule in every synthesis-side class the fold can invoke; + /// that is the memo's correctness contract, so this key-completeness audit has to be re-run whenever a + /// Synthesis*.cs rule or Allomorph/Morpher.IsWordValid changes. + /// + /// Not a starting point: SynthesisProbe's P1c fingerprint. That fingerprint is a + /// *measurement* key (docs/hermitcrab-synthesis-fold-probes.md section 3) and is unsound as a memo key + /// for exactly one reason, recorded as a trap in the plan doc and reconfirmed by the N1 census + /// retraction (section 7): it carries PendingTrailPosition, an integer index, and no + /// remaining-trail *content*. Two candidates at the same index with different pending rule sequences + /// compare equal under it. For a per-step measurement that is fine -- the "applied rule" half of + /// FoldStepKey already tells you which single rule was attempted. For a memo that *skips real + /// work and hands back a stored result*, it silently drops the distinct continuation the omitted trail + /// content represents: two states that will diverge on the very next step get merged, and the rarer + /// path's parse is lost. This key fixes exactly that gap (see below) and is + /// otherwise structurally close to the P1c fingerprint, because that fingerprint's own field list is + /// itself the result of an audit against these same classes. + /// + /// + /// Not Word.ValueEquals. (Word.cs) compares shape, realizational FS, non-heads, + /// stratum, root allomorph, trail, index and the final-rule flag, and omits + /// SyntacticFeatureStruct, MPR features, and disjunctive allomorph indices -- all three read + /// below. It also includes non-heads, which nothing this memo covers reads or writes (see field list). + /// + /// + /// Field-by-field justification, audited against , + /// , , + /// , , + /// , and the private Morpher.IsWordValid(Word): + /// + /// Shape (, compared/hashed via ValueEquals/ + /// GetFrozenHashCode) -- every rule's pattern match reads it + /// (SynthesisAffixProcessAllomorphRuleSpec), and Allomorph.IsWordValid's environment and + /// disjunctive-allomorph checks read the surrounding shape context. Shape.ValueEquals recurses + /// into every annotation's FeatureStruct, including each "Morph" annotation's MorphID and + /// Allomorph feature values -- so two candidates whose shape differs only in which + /// _mruleAppCount-derived MorphID got stamped on an otherwise-identical morph (a real hazard: + /// realizational rules are trail-exempt and can fire a different number of times along two paths that + /// otherwise converge) are already caught here, with no separate field needed. This is also why + /// Allomorphs/ObligatorySyntacticFeatures (both read by Morpher.IsWordValid) do not + /// need their own field: the allomorph-ID set embedded in every Morph annotation's FeatureStruct pins + /// _allomorphs' key set (and, since Allomorph objects are grammar-level singletons per ID, + /// its values too), and ObligatorySyntacticFeatures is a deterministic union of + /// rule.ObligatorySyntacticFeatures over the rules with a positive count in + /// , which is already a field below. + /// SyntacticFeatureStruct -- read by every rule's RequiredSyntacticFeatureStruct.Unify + /// (SynthesisAffixProcessRule.cs, SynthesisRealizationalAffixProcessRule.cs), by + /// SynthesisAffixTemplatesRule's IsUnifiable/ChooseInflectionalStem checks, and by + /// Morpher.IsWordValid's RealizationalFeatureStruct.IsUnifiable and obligatory-feature + /// checks. Hashed with a deliberately weak, freeze-free hash (mirroring + /// SynthesisProbe.SyntacticFeatureStructWeakHash): Word.FreezeImpl does not freeze this + /// field (AnalysisAffixTemplateRule.Apply mutates it on already-frozen Words), so + /// GetFrozenHashCode is unavailable and must not be forced by freezing it here as a side effect. + /// Correctness lives entirely in , which always does the real + /// FeatureStruct.ValueEquals; a hash collision only costs a linear bucket scan. + /// RealizationalFeatureStruct -- read by SynthesisRealizationalAffixProcessRule's + /// Subsumes/IsBlocked checks, by SynthesisAffixTemplatesRule.ChooseInflectionalStem + /// and its own IsUnifiable gate, and by Morpher.IsWordValid's IsUnifiable check. + /// MprFeatures -- read by every allomorph's RequiredMprFeatures/ExcludedMprFeatures + /// check in both memoized rule classes. + /// RootAllomorph (reference identity) -- SynthesisAffixProcessRule's + /// RequiredStemName check reads input.RootAllomorph.StemName directly, and + /// SynthesisAffixTemplatesRule.ChooseInflectionalStem reads + /// input.RootAllomorph.Morpheme's family/stratum. + /// DisjunctiveAllomorphIndices -- read by Allomorph.IsWordValid via + /// GetDisjunctiveAllomorphApplications; two words differing only here can validly disagree on + /// whether a later allomorph is blocked. Compared as a dictionary of sets (order-independent both + /// within each set and across morph IDs), matching how it is written + /// (Word.MorphologicalRuleApplied's UnionWith). + /// AppliedRuleCounts -- backs every rule's MaxApplicationCount gate + /// (SynthesisAffixProcessRule.cs) and realizational's "at most once" gate + /// (SynthesisRealizationalAffixProcessRule.cs), and -- see the Shape item above -- transitively + /// pins ObligatorySyntacticFeatures. Compared as a dictionary (order-independent), matching how + /// unapplication counts are compared on the analysis side. + /// IsPartial -- read by SynthesisAffixProcessRule's final-template-adjacency gates + /// and by SynthesisAffixTemplatesRule's applicable-template / no-template-fired branches. + /// IsLastAppliedRuleFinal -- read by the same final-template-adjacency gates and by + /// SynthesisStratumRule.Apply's own final-rule check + /// (mruleOutWord.IsLastAppliedRuleFinal ?? false). + /// Stratum -- SynthesisStratumRule.Apply gates on + /// input.RootAllomorph.Morpheme.Stratum.Depth > _stratum.Depth, and + /// HasRemainingRulesFromStratum reads it via curRule.Stratum. + /// Pending trail content (, an ORDERED sequence, not a + /// multiset -- unlike 's unapplication counts, order here is exactly what + /// determines which rule fires next, so two equal-length pending trails with the rules in a different + /// order are genuinely different states) -- IsMorphologicalRuleApplicable and + /// HasRemainingRulesFromStratum read only its first (current) entry to decide *this* step, but + /// the memo also has to guarantee the *next* step -- run against this step's stored output, replayed + /// onto a different query candidate -- sees the correct rest of the trail. This is the field the P1c + /// fingerprint deliberately omits; see the class remarks above. + /// + /// Deliberately excluded, with the reason a rule-read audit does not license including them: + /// + /// The already-consumed trail suffix (entries past the pending prefix) and the non-head list -- + /// never read by any rule again once passed, only by MorphemesInApplicationOrder on a finished + /// result. splices both from the query candidate rather than + /// the stored one, exactly because the key does not (and must not, to keep sharing meaningful) pin them + /// down. + /// _mruleAppCount and MorphID strings -- bookkeeping for a per-word morph-annotation + /// ordinal, not a decision input to any rule; already transitively pinned by Shape equality (see the + /// Shape item above), so adding it explicitly would be redundant, not more complete. + /// Compounding-rule state (CurrentNonHead/non-heads) -- read only by + /// , which this memo does not intercept (it is + /// out of the audited class list above, and out of what SynthesisProbe's P1c ratio measured). + /// Compounding-driven fold steps always fall through unmemoized. + /// + /// + /// + internal readonly struct SynthesisStateKey : IEquatable + { + private readonly Shape _shape; + private readonly FeatureStruct _syntacticFS; + private readonly FeatureStruct _realizationalFS; + private readonly MprFeatureSet _mprFeatures; + private readonly RootAllomorph _rootAllomorph; + private readonly IReadOnlyDictionary> _disjunctiveAllomorphIndices; + private readonly IReadOnlyDictionary _appliedRuleCounts; + private readonly bool _isPartial; + private readonly bool? _isLastAppliedRuleFinal; + private readonly Stratum _stratum; + private readonly IMorphologicalRule[] _pendingTrail; + private readonly int _hashCode; + + /// + /// Keys . A named factory to match AnalysisStateKey.PinAndKey's + /// style, though unlike that key this one never mutates : it deliberately + /// avoids freezing SyntacticFeatureStruct (see the class remarks). + /// + public static SynthesisStateKey PinAndKey(Word word) + { + return new SynthesisStateKey(word); + } + + private SynthesisStateKey(Word word) + { + if (!word.IsFrozen) + throw new ArgumentException( + "The word must be frozen before it can be used as a memo key.", + nameof(word) + ); + + _shape = word.Shape; + _syntacticFS = word.SyntacticFeatureStruct; + _realizationalFS = word.RealizationalFeatureStruct; + _mprFeatures = word.MprFeatures; + _rootAllomorph = word.RootAllomorph; + _disjunctiveAllomorphIndices = word.DisjunctiveAllomorphIndices; + _appliedRuleCounts = word.AppliedRuleCounts; + _isPartial = word.IsPartial; + _isLastAppliedRuleFinal = word.IsLastAppliedRuleFinal; + _stratum = word.Stratum; + + int pendingLength = word.PendingTrailPosition + 1; + _pendingTrail = pendingLength <= 0 ? Array.Empty() : new IMorphologicalRule[pendingLength]; + for (int i = 0; i < pendingLength; i++) + _pendingTrail[i] = word.MorphologicalRuleTrail[i]; + + _realizationalFS.Freeze(); + + int hash = 17; + hash = hash * 31 + _shape.GetFrozenHashCode(); + hash = hash * 31 + SyntacticFeatureStructWeakHash(_syntacticFS); + hash = hash * 31 + _realizationalFS.GetFrozenHashCode(); + hash = hash * 31 + UnorderedSetHash(_mprFeatures); + hash = hash * 31 + (_rootAllomorph?.GetHashCode() ?? 0); + hash = hash * 31 + UnorderedDictHash(_disjunctiveAllomorphIndices, UnorderedSetHash); + hash = hash * 31 + UnorderedDictHash(_appliedRuleCounts, v => v); + hash = hash * 31 + _isPartial.GetHashCode(); + hash = hash * 31 + _isLastAppliedRuleFinal.GetHashCode(); + hash = hash * 31 + (_stratum?.GetHashCode() ?? 0); + foreach (IMorphologicalRule rule in _pendingTrail) + hash = hash * 31 + (rule?.GetHashCode() ?? 0); + _hashCode = hash; + } + + public override int GetHashCode() => _hashCode; + + public override bool Equals(object obj) => obj is SynthesisStateKey other && Equals(other); + + public bool Equals(SynthesisStateKey other) + { + if (_hashCode != other._hashCode) + return false; + if (_isPartial != other._isPartial || _isLastAppliedRuleFinal != other._isLastAppliedRuleFinal) + return false; + if (!ReferenceEquals(_stratum, other._stratum) || !ReferenceEquals(_rootAllomorph, other._rootAllomorph)) + return false; + if (!PendingTrailEqual(_pendingTrail, other._pendingTrail)) + return false; + if (!_shape.ValueEquals(other._shape)) + return false; + if (!_syntacticFS.ValueEquals(other._syntacticFS) || !_realizationalFS.ValueEquals(other._realizationalFS)) + return false; + if (!_mprFeatures.SetEquals(other._mprFeatures)) + return false; + if (!DictEquals(_appliedRuleCounts, other._appliedRuleCounts, (x, y) => x == y)) + return false; + return DictEquals( + _disjunctiveAllomorphIndices, + other._disjunctiveAllomorphIndices, + (x, y) => x.SetEquals(y) + ); + } + + private static bool PendingTrailEqual(IMorphologicalRule[] a, IMorphologicalRule[] b) + { + if (a.Length != b.Length) + return false; + for (int i = 0; i < a.Length; i++) + { + if (a[i] != b[i]) + return false; + } + return true; + } + + // FeatureStruct.GetFrozenHashCode() throws unless frozen, and SyntacticFeatureStruct is + // deliberately never frozen by this key (see class remarks). Weak but always-safe: Equals always + // does the real ValueEquals, so a collision only costs a linear bucket scan, never a false merge. + private static int SyntacticFeatureStructWeakHash(FeatureStruct fs) + { + int acc = 0; + foreach (Feature f in fs.Features) + acc ^= f.GetHashCode(); + return acc; + } + + private static int UnorderedSetHash(IEnumerable items) + { + int acc = 0; + foreach (T item in items) + acc ^= item?.GetHashCode() ?? 0; + return acc; + } + + private static int UnorderedDictHash( + IReadOnlyDictionary dict, + Func valueHash + ) + { + int acc = 0; + foreach (KeyValuePair kvp in dict) + acc ^= ((kvp.Key?.GetHashCode() ?? 0) * 397) ^ valueHash(kvp.Value); + return acc; + } + + private static bool DictEquals( + IReadOnlyDictionary a, + IReadOnlyDictionary b, + Func valueEquals + ) + { + if (a.Count != b.Count) + return false; + foreach (KeyValuePair kvp in a) + { + if (!b.TryGetValue(kvp.Key, out TValue otherValue) || !valueEquals(kvp.Value, otherValue)) + return false; + } + return true; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index 4ca2c0c2..10437746 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -96,6 +96,7 @@ private Word(Word word, bool cloneNonHeadApps) _isPartial = word._isPartial; CurrentTrace = word.CurrentTrace; AnalysisScope = word.AnalysisScope; + SynthesisFoldScope = word.SynthesisFoldScope; _disjunctiveAllomorphIndices = word._disjunctiveAllomorphIndices.ToDictionary( kvp => kvp.Key, kvp => new HashSet(kvp.Value) @@ -228,6 +229,18 @@ public IEnumerable MorphemesInApplicationOrder /// internal AnalysisScope AnalysisScope { get; set; } + /// + /// Carrier for the synthesis fold-step memo (docs/hermitcrab-synthesis-fold-probes.md). Same + /// contract as : reference-shared through clones, excluded from + /// FreezeImpl/ValueEquals, null unless is + /// on and the parse is running sequentially and untraced. One instance per + /// call, installed on every alternative that + /// enters _synthesisRule.Apply in Morpher.SynthesizeSequential -- shared across every + /// analysis word's alternatives for that one surface-word parse, which is exactly the scope P1c/N1 + /// measured sharing over. + /// + internal SynthesisFoldScope SynthesisFoldScope { get; set; } + public bool IsPartial { get { return _isPartial; } @@ -429,6 +442,18 @@ internal int GetApplicationCount(IMorphologicalRule mrule) /// internal int PendingTrailPosition => _mruleAppIndex; + /// + /// The full morphological-rule trail list, for . Read together with + /// : entries at indices 0..PendingTrailPosition are the + /// still-pending trail content a synthesis step's continuation depends on (what + /// 's P1c fingerprint omits -- position only, no content, see the plan + /// doc's trap #1); entries past PendingTrailPosition are already consumed and never read + /// again by any rule, only by on a fully-finished result. + /// This list itself never mutates during synthesis -- only + /// moves -- so exposing the live list is safe. + /// + internal IReadOnlyList MorphologicalRuleTrail => _mruleApps; + internal Word CurrentNonHead { get @@ -573,6 +598,88 @@ internal List CloneNonHeadsForReplay() return new List(_nonHeadApps.CloneItems()); } + /// + /// Grafts a memoized synthesis fold-step result (this, produced by applying one rule to some + /// storedInput) onto , whose + /// equals storedInput's. The single-step mirror of -- see + /// docs/hermitcrab-synthesis-fold-probes.md, "What to build" item 2. + /// + /// Sound because equality guarantees storedInput and + /// agree on everything the step read (Shape -- including every morph + /// annotation's MorphID/Allomorph feature values, so this word's own already-marked + /// _allomorphs and the new step's MorphID stamp are already correct as computed -- both + /// FeatureStructs, MprFeatures, disjunctive-allomorph indices, per-rule applied counts, IsPartial, + /// IsLastAppliedRuleFinal, Stratum, and the pending trail content) and on everything that field + /// determines transitively (ObligatorySyntacticFeatures is a deterministic union over which + /// rules have applied at least once, i.e. over AppliedRuleCounts' support). So this word's + /// own computed content carries over UNCHANGED: it is exactly what re-running the step on + /// would have produced too. + /// + /// + /// Two things do NOT transfer, because they are exactly what the key -- correctly -- does not pin + /// down, and because neither of the two rule classes this memo covers + /// (, + /// ) touches them: + /// + /// The already-consumed suffix of the morphological-rule trail (entries past + /// 's ). No rule reads it again, but + /// MorphemesInApplicationOrder walks the whole list on a finished result, so it must be + /// 's own history, not the stored candidate's. + /// The non-head list -- neither memoized rule class calls NonHeadUnapplied, so it must + /// stay exactly what already had. + /// + /// Both are spliced from , this word's own copies discarded. + /// + /// + /// Blocking exception. Apply's CheckBlocking branch can replace a step's output + /// with a wholly fresh built from a sibling LexEntry (), + /// which is born with an EMPTY morphological-rule trail and no reference to the input that produced + /// it at all -- it is not a continuation, it overrides one. Reaching this method's call sites only + /// happens once has already required a non-empty trail on + /// the real input, so an empty trail on this can only mean blocking fired, never a + /// coincidentally-empty ordinary continuation. Such a result is already correct for any + /// same-key query and is returned as-is, unspliced: grafting 's trail + /// onto it would fabricate morphemes MorphemesInApplicationOrder was never meant to report + /// and could make Morpher.IsWordValid's IsAllMorphologicalRulesApplied check reject a + /// word the unmemoized engine would have accepted. + /// + /// + /// + /// The word that hit the memo. Its trail (advanced by exactly this step) and non-heads become this + /// result's identity. + /// + /// + /// True for a trail-driven rule (), which + /// always advances by one once + /// has already gated entry; false for a trail-exempt + /// realizational rule (), + /// which never does. Mirrors exactly what itself would have + /// done to . + /// + internal Word ReanchorSynthesisStep(Word queryInput, bool trailConsuming) + { + // See the "Blocking exception" remarks above. + if (_mruleApps.Count == 0) + return this; + + var clone = new Word(this, cloneNonHeadApps: false); + + clone._mruleApps.Clear(); + clone._mruleApps.AddRange(queryInput._mruleApps); + clone._mruleAppIndex = trailConsuming ? queryInput._mruleAppIndex - 1 : queryInput._mruleAppIndex; + + clone._nonHeadApps.AddRange(queryInput._nonHeadApps.CloneItems()); + clone._nonHeadAppIndex = queryInput._nonHeadAppIndex; + + clone.CurrentTrace = queryInput.CurrentTrace; + clone.AnalysisScope = queryInput.AnalysisScope; + clone.SynthesisFoldScope = queryInput.SynthesisFoldScope; + clone.Source = queryInput; + + clone.Freeze(); + return clone; + } + public Allomorph GetAllomorph(Annotation morph) { var alloID = (string)morph.FeatureStruct.GetValue(HCFeatureSystem.Allomorph); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldMemoVerification.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldMemoVerification.cs new file mode 100644 index 00000000..1b52c2b4 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SynthesisFoldMemoVerification.cs @@ -0,0 +1,368 @@ +#nullable disable +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Memo-on/memo-off equality for +/// (docs/hermitcrab-synthesis-fold-probes.md), the synthesis-side counterpart to +/// . Both sides run at maxDegreeOfParallelism: 1 -- the toggle +/// only takes effect there -- so the only variable is the toggle itself. +/// +/// Two gates: the 33 committed conformance fixtures (fast, synthetic, safe to run unconditionally) and a +/// real corpus via the same env vars uses (this repo never commits +/// real grammars or word lists). [Explicit] throughout, matching every other corpus/fixture-sweep test in +/// this file's neighbourhood (, ): not +/// part of CI, run manually. +/// +/// +/// dotnet test --filter "FullyQualifiedName~SynthesisFoldMemoVerification.MemoOnMatchesMemoOff_AnalysisSetIdentical_AcrossConformanceFixtures" +/// +/// $env:HC_MEMO_GRAMMAR = "...\sena-hc.xml" +/// $env:HC_MEMO_WORDS = "...\sena-words.txt" +/// $env:HC_MEMO_MAX_WORDS = "60" +/// $env:HC_MEMO_TIMEOUT_MS = "600000" +/// dotnet test --filter "FullyQualifiedName~SynthesisFoldMemoVerification.MemoOnMatchesMemoOff_AnalysisSetIdentical_OnRealCorpus" +/// +/// +[TestFixture] +[Explicit("Manual corpus/fixture verification; not part of CI. See docs/hermitcrab-synthesis-fold-probes.md.")] +public class SynthesisFoldMemoVerification +{ + [Test] + public void MemoOnMatchesMemoOff_AnalysisSetIdentical_AcrossConformanceFixtures() + { + string fixturesRoot = Environment.GetEnvironmentVariable("HC_PROBE_FIXTURES_ROOT"); + if (string.IsNullOrEmpty(fixturesRoot)) + fixturesRoot = Path.Combine(RepositoryRoot(), "conformance"); + + List fixtures = Fixture.DiscoverAll(fixturesRoot); + Assert.That(fixtures, Is.Not.Empty, $"no fixtures discovered under {fixturesRoot}"); + + var divergences = new List(); + long totalHitsAcrossFixtures = 0; + var fixtureTimings = new List<(string Id, double OnMs, double OffMs, int Words)>(); + + foreach (Fixture fixture in fixtures) + { + Language language; + try + { + language = XmlLanguageLoader.Load(fixture.GrammarPath); + } + catch (Exception e) + { + TestContext.Out.WriteLine($"[{fixture.Id}] grammar failed to load: {e.GetType().Name} -- skipped"); + continue; + } + + var memoOff = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + var memoOn = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + UseSynthesisFoldMemo = true, + }; + + double onMs = 0; + double offMs = 0; + foreach (WordEntry entry in fixture.Words.Words) + { + var swOff = Stopwatch.StartNew(); + List offSignatures = Signatures(memoOff, entry.Word); + swOff.Stop(); + offMs += swOff.Elapsed.TotalMilliseconds; + + var swOn = Stopwatch.StartNew(); + List onSignatures = Signatures(memoOn, entry.Word); + swOn.Stop(); + onMs += swOn.Elapsed.TotalMilliseconds; + + if (!onSignatures.SequenceEqual(offSignatures)) + { + divergences.Add( + $"[{fixture.Id}] {entry.Word}: memo-on={{{string.Join(",", onSignatures)}}} vs " + + $"memo-off={{{string.Join(",", offSignatures)}}}" + ); + } + } + + totalHitsAcrossFixtures += memoOn.SynthesisFoldHits; + fixtureTimings.Add((fixture.Id, onMs, offMs, fixture.Words.Words.Count)); + TestContext.Out.WriteLine( + $"[{fixture.Id}] words={fixture.Words.Words.Count} memo-on={onMs:F2}ms memo-off={offMs:F2}ms " + + $"hits={memoOn.SynthesisFoldHits} " + + $"speedup={(onMs > 0 ? offMs / onMs : 0):F2}x" + ); + } + + TestContext.Out.WriteLine($"total synthesis fold-memo hits across all fixtures: {totalHitsAcrossFixtures}"); + TestContext.Out.WriteLine("--- the two reliable fixtures from section 8 of the plan doc ---"); + foreach ( + string id in new[] + { + "edge-cases/deep-optional-affix-nesting", + "languages/suffixing-evidential-adjacency-chain", + } + ) + { + var row = fixtureTimings.FirstOrDefault(r => r.Id == id); + if (row.Id == null) + { + TestContext.Out.WriteLine($"{id}: not found among discovered fixtures"); + continue; + } + double speedup = row.OnMs > 0 ? row.OffMs / row.OnMs : 0; + double wallSaved = row.OffMs > 0 ? (1 - row.OnMs / row.OffMs) * 100 : 0; + TestContext.Out.WriteLine( + $"{id}: memo-off={row.OffMs:F2}ms memo-on={row.OnMs:F2}ms speedup={speedup:F2}x " + + $"wall-clock-reduction={wallSaved:F1}%" + ); + } + + Assert.That( + divergences, + Is.Empty, + $"{divergences.Count} word(s) diverged between memo-on and memo-off across the 33 conformance " + + $"fixtures (showing up to 10): {string.Join(" | ", divergences.Take(10))}" + ); + } + + // One discarded warm-up per arm per word, then MeasuredReps interleaved samples of which the MINIMUM + // is kept, on FRESH Morphers built just for this test -- the correctness sweep above reuses one + // Morpher per fixture across every word, which is fine for equality but leaves memo tables/JIT state + // that would bias a single-pass timing comparison. Interleaving off/on per rep, rather than running + // all off reps then all on reps, means a one-off GC or JIT stall lands on both arms rather than + // whichever ran first. section 6.1 of the plan doc found sub-2ms fixtures unreliable to time at all; + // this is the section-8-style rigor those two numbers specifically need. + private const int WarmupReps = 1; + private const int MeasuredReps = 5; + + [Test] + public void MeasuredSpeedup_OnTheTwoReliableFixtures() + { + string fixturesRoot = Environment.GetEnvironmentVariable("HC_PROBE_FIXTURES_ROOT"); + if (string.IsNullOrEmpty(fixturesRoot)) + fixturesRoot = Path.Combine(RepositoryRoot(), "conformance"); + List fixtures = Fixture.DiscoverAll(fixturesRoot); + + foreach ( + string id in new[] + { + "edge-cases/deep-optional-affix-nesting", + "languages/suffixing-evidential-adjacency-chain", + } + ) + { + Fixture fixture = fixtures.FirstOrDefault(f => f.Id == id); + if (fixture == null) + { + TestContext.Out.WriteLine($"{id}: not found among discovered fixtures"); + continue; + } + + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + var off = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + var on = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + UseSynthesisFoldMemo = true, + }; + + for (int w = 0; w < WarmupReps; w++) + { + foreach (WordEntry entry in fixture.Words.Words) + { + Signatures(off, entry.Word); + Signatures(on, entry.Word); + } + } + + double offMin = double.MaxValue; + double offMax = 0; + double onMin = double.MaxValue; + for (int r = 0; r < MeasuredReps; r++) + { + var swOff = Stopwatch.StartNew(); + foreach (WordEntry entry in fixture.Words.Words) + Signatures(off, entry.Word); + swOff.Stop(); + + var swOn = Stopwatch.StartNew(); + foreach (WordEntry entry in fixture.Words.Words) + Signatures(on, entry.Word); + swOn.Stop(); + + double offMs = swOff.Elapsed.TotalMilliseconds; + double onMs = swOn.Elapsed.TotalMilliseconds; + offMin = Math.Min(offMin, offMs); + offMax = Math.Max(offMax, offMs); + onMin = Math.Min(onMin, onMs); + } + + double speedup = onMin > 0 ? offMin / onMin : 0; + double wallSaved = offMin > 0 ? (1 - onMin / offMin) * 100 : 0; + // Off-arm (max - min) as a percentage of the off-arm floor: how much of any apparent delta + // could just be noise. A speedup implying less wall-clock reduction than this figure is not a + // result. + double offSpreadPct = offMin > 0 ? (offMax - offMin) / offMin * 100 : 0; + TestContext.Out.WriteLine( + $"{id}: memo-off(min of {MeasuredReps})={offMin:F2}ms memo-on(min of {MeasuredReps})={onMin:F2}ms " + + $"speedup={speedup:F2}x wall-clock-reduction={wallSaved:F1}% " + + $"off-arm-spread={offSpreadPct:F1}% hits={on.SynthesisFoldHits}" + ); + } + } + + [Test] + public void MemoOnMatchesMemoOff_AnalysisSetIdentical_OnRealCorpus() + { + (Language language, List words) = Load(); + + var memoOff = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + var memoOn = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + UseSynthesisFoldMemo = true, + }; + int timeoutMs = int.TryParse(Environment.GetEnvironmentVariable("HC_MEMO_TIMEOUT_MS"), out int t) + ? t + : 5000; + + var perWordTimes = new List<(string Word, double OnMs, double OffMs)>(); + var divergences = new List(); + var timedOut = new List(); + int noParseBoth = 0; + + foreach (string word in words) + { + List onSignatures; + List offSignatures; + double onMs; + double offMs; + try + { + var swOff = Stopwatch.StartNew(); + offSignatures = RunWithTimeout(() => Signatures(memoOff, word), timeoutMs); + swOff.Stop(); + offMs = swOff.Elapsed.TotalMilliseconds; + + var swOn = Stopwatch.StartNew(); + onSignatures = RunWithTimeout(() => Signatures(memoOn, word), timeoutMs); + swOn.Stop(); + onMs = swOn.Elapsed.TotalMilliseconds; + } + catch (TimeoutException) + { + timedOut.Add(word); + continue; + } + perWordTimes.Add((word, onMs, offMs)); + + if (onSignatures.Count == 0 && offSignatures.Count == 0) + noParseBoth++; + + if (!onSignatures.SequenceEqual(offSignatures)) + { + divergences.Add( + $"{word}: memo-on={{{string.Join(",", onSignatures)}}} vs " + + $"memo-off={{{string.Join(",", offSignatures)}}}" + ); + } + + TestContext.Out.WriteLine( + $"{word}: memo-off={offMs:F1}ms memo-on={onMs:F1}ms " + + $"speedup={(onMs > 0 ? offMs / onMs : 0):F2}x hits-so-far={memoOn.SynthesisFoldHits}" + ); + } + + double totalOnMs = perWordTimes.Sum(x => x.OnMs); + double totalOffMs = perWordTimes.Sum(x => x.OffMs); + TestContext.Out.WriteLine($"words attempted: {words.Count}, timed out (>{timeoutMs}ms): {timedOut.Count}"); + TestContext.Out.WriteLine($"words with no parse on both sides: {noParseBoth}"); + TestContext.Out.WriteLine( + $"wall-clock: memo-on total {totalOnMs:F1} ms vs memo-off total {totalOffMs:F1} ms " + + $"({(totalOnMs > 0 ? totalOffMs / totalOnMs : 0):F2}x)" + ); + TestContext.Out.WriteLine($"synthesis fold-memo hits (final Morpher totals): {memoOn.SynthesisFoldHits}"); + if (timedOut.Count > 0) + { + TestContext.Out.WriteLine( + $"timed-out words (excluded from the equality gate above): {string.Join(", ", timedOut)}" + ); + } + + Assert.That( + divergences, + Is.Empty, + $"{divergences.Count} word(s) diverged between memo-on and memo-off " + + $"(showing up to 10): {string.Join(" | ", divergences.Take(10))}" + ); + } + + // Some conformance fixtures (e.g. edge-cases/simultaneous-epenthesis-cascade) are deliberately built + // so that a CORRECT C# engine throws -- InvalidShapeException for an out-of-inventory character (as + // Morpher.AnalyzeWord already tolerates) and InfiniteLoopException for a rewrite-rule runaway the + // engine's own hard cap catches. Neither is a memo concern: both are thrown well outside the + // memoized region (character-table lookup and post-fold phonological rewriting respectively). Rather + // than special-case fixture ids, any such crash is folded into a one-element sentinel signature so + // memo-on/memo-off comparison still works uniformly -- if only one side crashes, or the two sides + // crash with different exception types, that sentinel mismatch surfaces as a real divergence. + private static List Signatures(Morpher morpher, string word) + { + try + { + return morpher + .ParseWord(word) + .Select(MorpherTests.WordAnalysisSignature) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + } + catch (InvalidShapeException) + { + return new List(); + } + catch (Exception e) when (e is InfiniteLoopException or global::SIL.Machine.Rules.MaxAlternativesExceededException) + { + return new List { $"<>" }; + } + } + + // See MemoCorpusVerification.RunWithTimeout for why this cannot cooperatively cancel. + private static T RunWithTimeout(Func action, int timeoutMs) + { + Task task = Task.Run(action); + if (!task.Wait(timeoutMs)) + throw new TimeoutException(); + return task.Result; + } + + private static string RepositoryRoot() + { + string directory = TestContext.CurrentContext.TestDirectory; + while (directory != null) + { + if (File.Exists(Path.Combine(directory, "conformance", "constructs.txt"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root."); + return string.Empty; + } + + private static (Language, List) Load() + { + string grammarPath = Environment.GetEnvironmentVariable("HC_MEMO_GRAMMAR"); + string wordsPath = Environment.GetEnvironmentVariable("HC_MEMO_WORDS"); + if (string.IsNullOrEmpty(grammarPath) || string.IsNullOrEmpty(wordsPath)) + Assert.Ignore("set HC_MEMO_GRAMMAR and HC_MEMO_WORDS"); + + int maxWords = int.TryParse(Environment.GetEnvironmentVariable("HC_MEMO_MAX_WORDS"), out int mw) ? mw : 60; + Language language = XmlLanguageLoader.Load(grammarPath); + List words = File.ReadAllLines(wordsPath) + .Select(w => w.Trim()) + .Where(w => w.Length > 0) + .Take(maxWords) + .ToList(); + return (language, words); + } +}