Convert USFM versification

src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs

@@ -0,0 +1,151 @@
+using System.Collections.Generic;
+using System.Linq;
+using SIL.Scripture;
+
+namespace SIL.Machine.Corpora
+{
+    public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase
+    {
+        private readonly List<UsfmToken> _tokens;
+        private VerseRef _prevVerseRef;
+        private int _verseBoundary;
+        private readonly ScrVers _targetVersification;
+        private int _insertChapterIndex;
+        private bool _skip;
+
+        public ConvertUsfmVersificationHandler(ScrVers targetVersification)
+        {
+            _verseBoundary = 0;
+            _insertChapterIndex = -1;
+            _tokens = new List<UsfmToken>();
+            _prevVerseRef = new VerseRef();
+            _targetVersification = targetVersification;
+            _skip = false;
+        }
+
+        public override void Chapter(
+            UsfmParserState state,
+            string number,
+            string marker,
+            string altNumber,
+            string pubNumber
+        )
+        {
+            base.Chapter(state, number, marker, altNumber, pubNumber);
+            ProcessTokens(state);
+            _insertChapterIndex = _tokens.Count;
+        }
+
+        public override void Verse(
+            UsfmParserState state,
+            string number,
+            string marker,
+            string altNumber,
+            string pubNumber
+        )
+        {
+            base.Verse(state, number, marker, altNumber, pubNumber);
+
+            VerseRef verseRef = state.VerseRef;
+
+            ProcessTokens(state);
+
+            List<VerseRef> verseRefs = state
+                .VerseRef.AllVerses()
+                .Select(vr => vr.ChangeVersificationWithSegments(_targetVersification))
+                .ToList();
+
+            if (
+                _prevVerseRef.IsDefault
+                || (
+                    verseRefs[0].BookNum == _prevVerseRef.BookNum && verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum
+                )
+            )
+            {
+                UsfmToken newChapterToken = new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[0].Chapter);
+
+                if (_insertChapterIndex == -1)
+                    _tokens.Add(newChapterToken);
+                else
+                    _tokens.Insert(_insertChapterIndex, newChapterToken);
+            }
+
+            string start = null;
+            for (int i = 0; i < verseRefs.Count; i++)
+            {
+                if (!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book)
+                {
+                    continue;
+                }
+                if (start != null)
+                {
+                    string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : "";
+                    if (
+                        _prevVerseRef.BookNum == verseRefs[i].BookNum
+                        && _prevVerseRef.ChapterNum != verseRefs[i].ChapterNum
+                    )
+                    {
+                        _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
+                        _tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter));
+                        start = verseRefs[i].Verse;
+                        _prevVerseRef = verseRefs[i];
+                    }
+                    else if (_prevVerseRef.VerseNum + 1 != verseRefs[i].VerseNum)
+                    {
+                        _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
+                        start = verseRefs[i].Verse;
+                        _prevVerseRef = verseRefs[i];
+                    }
+                    else
+                    {
+                        _prevVerseRef = verseRefs[i];
+                    }
+                }
+                else
+                {
+                    start = verseRefs[i].Verse;
+                    _prevVerseRef = verseRefs[i];
+                }
+                verseRef = verseRefs[i];
+            }
+
+            if (start != null)
+            {
+                string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : "";
+                _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
+                _skip = false;
+                _insertChapterIndex = -1;
+                _prevVerseRef = verseRef;
+            }
+            else
+            {
+                _skip = true;
+            }
+        }
+
+        public override void EndUsfm(UsfmParserState state)
+        {
+            base.EndUsfm(state);
+            ProcessTokens(state);
+            if (!_skip && !(state.Token.Type == UsfmTokenType.Chapter || state.Token.Type == UsfmTokenType.Verse))
+                _tokens.Add(state.Token);
+        }
+
+        public string GetUsfm(UsfmStylesheet stylesheet)
+        {
+            var tokenizer = new UsfmTokenizer(stylesheet);
+            return tokenizer.Detokenize(_tokens);
+        }
+
+        private void ProcessTokens(UsfmParserState state)
+        {
+            int offset = 0;
+            if (!_skip)
+            {
+                while (_verseBoundary + offset < state.Index)
+                    _tokens.Add(state.Tokens[_verseBoundary + offset++]);
+            }
+            _verseBoundary = state.Index + 1;
+        }
+    }
+}

src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using SIL.Scripture;
+
+namespace SIL.Machine.Corpora
+{
+    public abstract class ParatextProjectVersificationConverterBase
+    {
+        private readonly ParatextProjectSettings _settings;
+        private readonly IParatextProjectFileHandler _paratextProjectFileHandler;
+
+        protected ParatextProjectVersificationConverterBase(
+            IParatextProjectFileHandler paratextProjectFileHandler,
+            ParatextProjectSettings settings
+        )
+        {
+            _settings = settings;
+            _paratextProjectFileHandler = paratextProjectFileHandler;
+        }
+
+        public string UpdateUsfm(string bookId, ScrVers targetVersification)
+        {
+            string fileName = _settings.GetBookFileName(bookId);
+            if (!Exists(fileName))
+                return null;
+
+            string usfm;
+            using (var reader = new StreamReader(Open(fileName)))
+            {
+                usfm = reader.ReadToEnd();
+            }
+
+            var handler = new ConvertUsfmVersificationHandler(targetVersification);
+            try
+            {
+                var tokenizer = new UsfmTokenizer(_settings.Stylesheet);
+                IReadOnlyList<UsfmToken> tokens = tokenizer.Tokenize(usfm);
+                UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification);
+                return handler.GetUsfm(_settings.Stylesheet);
+            }
+            catch (Exception ex)
+            {
+                var sb = new StringBuilder();
+                sb.Append($"An error occurred while parsing the usfm for '{bookId}`");
+                if (!string.IsNullOrEmpty(_settings.Name))
+                    sb.Append($" in project '{_settings.Name}'");
+                sb.Append($". Error: '{ex.Message}'");
+                throw new InvalidOperationException(sb.ToString(), ex);
+            }
+        }
+
+        private bool Exists(string fileName) => _paratextProjectFileHandler.Exists(fileName);
+
+        private Stream Open(string fileName) => _paratextProjectFileHandler.Open(fileName);
+    }
+}