Move memory Paratext classes to testutils and remove filter_tokens_by_chapter export

- Move MemoryParatextProjectFileHandler and MemoryParatextProjectTextUpdater to tests/testutils
- Remove MemoryParatextProjectFileHandler and MemoryParatextProjectTextUpdater from machine/corpora
- Remove filter_tokens_by_chapter from public exports
- Update all imports to use testutils locations

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: mshannon-sil

machine/corpora/__init__.py

@@ -14,8 +14,6 @@
 from .file_paratext_project_versification_error_detector import FileParatextProjectVersificationErrorDetector
 from .flatten import flatten
 from .memory_alignment_collection import MemoryAlignmentCollection
-from .memory_paratext_project_file_handler import MemoryParatextProjectFileHandler
-from .memory_paratext_project_text_updater import MemoryParatextProjectTextUpdater
 from .memory_stream_container import MemoryStreamContainer
 from .memory_text import MemoryText
 from .multi_key_ref import MultiKeyRef
@@ -29,7 +27,7 @@
 from .paratext_project_settings import ParatextProjectSettings
 from .paratext_project_settings_parser_base import ParatextProjectSettingsParserBase
 from .paratext_project_terms_parser_base import KeyTerm, ParatextProjectTermsParserBase
-from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase, filter_tokens_by_chapter
+from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase
 from .paratext_project_versification_error_detector_base import ParatextProjectVersificationErrorDetectorBase
 from .paratext_text_corpus import ParatextTextCorpus
 from .place_markers_usfm_update_block_handler import PlaceMarkersAlignmentInfo, PlaceMarkersUsfmUpdateBlockHandler
@@ -117,14 +115,11 @@
     "FileParatextProjectTermsParser",
     "FileParatextProjectTextUpdater",
     "FileParatextProjectVersificationErrorDetector",
-    "filter_tokens_by_chapter",
     "flatten",
     "is_scripture",
     "KeyTerm",
     "lowercase",
     "MemoryAlignmentCollection",
-    "MemoryParatextProjectFileHandler",
-    "MemoryParatextProjectTextUpdater",
     "MemoryStreamContainer",
     "MemoryText",
     "MultiKeyRef",

machine/corpora/memory_paratext_project_file_handler.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional
 
-from testutils.default_paratext_project_settings import DefaultParatextProjectSettings
+from testutils.memory_paratext_project_file_handler import DefaultParatextProjectSettings
 from testutils.memory_paratext_project_terms_parser import MemoryParatextProjectTermsParser
 
 from machine.corpora import KeyTerm, ParatextProjectSettings, ParatextProjectTermsParserBase

machine/corpora/memory_paratext_project_text_updater.py

@@ -1,11 +1,10 @@
 from typing import Iterable, List, Optional, Sequence, Tuple, Union
 
 from testutils.corpora_test_helpers import USFM_TEST_PROJECT_PATH, ignore_line_endings
-from testutils.default_paratext_project_settings import DefaultParatextProjectSettings
+from testutils.memory_paratext_project_file_handler import DefaultParatextProjectSettings, MemoryParatextProjectTextUpdater
 
 from machine.corpora import (
     FileParatextProjectTextUpdater,
-    MemoryParatextProjectTextUpdater,
     ScriptureRef,
     UpdateUsfmMarkerBehavior,
     UpdateUsfmRow,

tests/corpora/test_paratext_project_terms_parser.py

@@ -1,7 +1,7 @@
 from io import StringIO
 from typing import Dict, List, Optional
 
-from testutils.default_paratext_project_settings import DefaultParatextProjectSettings
+from testutils.memory_paratext_project_file_handler import DefaultParatextProjectSettings
 from testutils.memory_paratext_project_versification_error_detector import (
     MemoryParatextProjectVersificationErrorDetector,
 )

tests/corpora/test_update_usfm_parser_handler.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional
 
-from testutils.default_paratext_project_settings import DefaultParatextProjectSettings
+from testutils.memory_paratext_project_file_handler import DefaultParatextProjectSettings
 from testutils.memory_paratext_project_quote_convention_detector import MemoryParatextProjectQuoteConventionDetector
 
 from machine.corpora import ParatextProjectSettings

tests/corpora/test_usfm_versification_error_detector.py

@@ -1,9 +1,36 @@
-from typing import Optional
+from io import BytesIO
+from typing import BinaryIO, Dict, Optional
 
-from machine.corpora import ParatextProjectSettings, UsfmStylesheet
+from machine.corpora import ParatextProjectFileHandler, ParatextProjectSettings, UsfmStylesheet
+from machine.corpora.paratext_project_text_updater_base import ParatextProjectTextUpdaterBase
 from machine.scripture import ORIGINAL_VERSIFICATION, Versification
 
 
+class MemoryParatextProjectFileHandler(ParatextProjectFileHandler):
+    def __init__(self, files: Dict[str, str]) -> None:
+        self.files = files
+
+    def exists(self, file_name: str) -> bool:
+        return file_name in self.files
+
+    def open(self, file_name: str) -> BinaryIO:
+        return BytesIO(self.files[file_name].encode("utf-8"))
+
+    def find(self, extension: str) -> Optional[str]:
+        for name in self.files:
+            if name.endswith(extension):
+                return name
+        return None
+
+    def create_stylesheet(self, file_name: str) -> UsfmStylesheet:
+        return UsfmStylesheet(file_name)
+
+
+class MemoryParatextProjectTextUpdater(ParatextProjectTextUpdaterBase):
+    def __init__(self, files: Dict[str, str], settings: ParatextProjectSettings) -> None:
+        super().__init__(MemoryParatextProjectFileHandler(files), settings)
+
+
 class DefaultParatextProjectSettings(ParatextProjectSettings):
     def __init__(
         self,

tests/punctuation_analysis/test_paratext_project_quote_convention_detector.py

@@ -1,9 +1,9 @@
 from typing import Dict
 
-from machine.corpora import MemoryParatextProjectFileHandler, ParatextProjectSettings
+from machine.corpora import ParatextProjectSettings
 from machine.punctuation_analysis import ParatextProjectQuoteConventionDetector
 
-from .default_paratext_project_settings import DefaultParatextProjectSettings
+from .memory_paratext_project_file_handler import DefaultParatextProjectSettings, MemoryParatextProjectFileHandler
 
 
 class MemoryParatextProjectQuoteConventionDetector(ParatextProjectQuoteConventionDetector):

…ils/default_paratext_project_settings.py …/memory_paratext_project_file_handler.pytests/testutils/default_paratext_project_settings.py renamed to tests/testutils/memory_paratext_project_file_handler.py tests/testutils/default_paratext_project_settings.py renamed to tests/testutils/memory_paratext_project_file_handler.py

@@ -1,8 +1,8 @@
 from typing import Dict, Optional
 
-from machine.corpora import MemoryParatextProjectFileHandler, ParatextProjectSettings, ParatextProjectTermsParserBase
+from machine.corpora import ParatextProjectSettings, ParatextProjectTermsParserBase
 
-from .default_paratext_project_settings import DefaultParatextProjectSettings
+from .memory_paratext_project_file_handler import DefaultParatextProjectSettings, MemoryParatextProjectFileHandler
 
 
 class MemoryParatextProjectTermsParser(ParatextProjectTermsParserBase):