Add back MSG parsing (#1993)

* Add back MSG parsing

* Fix init

* Fix msg parsing

py/core/__init__.py

@@ -132,7 +132,7 @@
     "EMLParser",
     "EPUBParser",
     "JSONParser",
-    # "MSGParser",
+    "MSGParser",
     "ORGParser",
     "P7SParser",
     "RSTParser",

py/core/parsers/__init__.py

@@ -21,7 +21,7 @@
     "EMLParser",
     "EPUBParser",
     "JSONParser",
-    # "MSGParser",
+    "MSGParser",
     "ORGParser",
     "P7SParser",
     "RSTParser",

py/core/parsers/structured/__init__.py

@@ -3,8 +3,7 @@
 from .eml_parser import EMLParser
 from .epub_parser import EPUBParser
 from .json_parser import JSONParser
-
-# from .msg_parser import MSGParser
+from .msg_parser import MSGParser
 from .org_parser import ORGParser
 from .p7s_parser import P7SParser
 from .rst_parser import RSTParser
@@ -19,7 +18,7 @@
     "EMLParser",
     "EPUBParser",
     "JSONParser",
-    # "MSGParser",
+    "MSGParser",
     "ORGParser",
     "P7SParser",
     "RSTParser",

py/core/parsers/structured/msg_parser.py

@@ -1,67 +1,65 @@
-# # type: ignore
-# from typing import AsyncGenerator
+# type: ignore
+import os
+import tempfile
+from typing import AsyncGenerator
 
-# import extract_msg
+from msg_parser import MsOxMessage
 
-# from core.base.parsers.base_parser import AsyncParser
-# from core.base.providers import (
-#     CompletionProvider,
-#     DatabaseProvider,
-#     IngestionConfig,
-# )
+from core.base.parsers.base_parser import AsyncParser
+from core.base.providers import (
+    CompletionProvider,
+    DatabaseProvider,
+    IngestionConfig,
+)
 
-# class MSGParser(AsyncParser[str | bytes]):
-#     """Parser for MSG (Outlook Message) files."""
 
-#     def __init__(
-#         self,
-#         config: IngestionConfig,
-#         database_provider: DatabaseProvider,
-#         llm_provider: CompletionProvider,
-#     ):
-#         self.database_provider = database_provider
-#         self.llm_provider = llm_provider
-#         self.config = config
-#         self.extract_msg = extract_msg
+class MSGParser(AsyncParser[str | bytes]):
+    """Parser for MSG (Outlook Message) files using msg_parser."""
 
-#     async def ingest(
-#         self, data: str | bytes, **kwargs
-#     ) -> AsyncGenerator[str, None]:
-#         """Ingest MSG data and yield email content."""
-#         if isinstance(data, str):
-#             raise ValueError("MSG data must be in bytes format.")
+    def __init__(
+        self,
+        config: IngestionConfig,
+        database_provider: DatabaseProvider,
+        llm_provider: CompletionProvider,
+    ):
+        self.database_provider = database_provider
+        self.llm_provider = llm_provider
+        self.config = config
 
-#         from io import BytesIO
+    async def ingest(
+        self, data: str | bytes, **kwargs
+    ) -> AsyncGenerator[str, None]:
+        """Ingest MSG data and yield email content."""
+        if isinstance(data, str):
+            raise ValueError("MSG data must be in bytes format.")
 
-#         file_obj = BytesIO(data)
+        tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".msg")
+        try:
+            tmp_file.write(data)
+            tmp_file.close()
 
-#         try:
-#             msg = self.extract_msg.Message(file_obj)
+            msg = MsOxMessage(tmp_file.name)
 
-#             # Extract metadata
-#             metadata = []
-#             if msg.subject:
-#                 metadata.append(f"Subject: {msg.subject}")
-#             if msg.sender:
-#                 metadata.append(f"From: {msg.sender}")
-#             if msg.to:
-#                 metadata.append(f"To: {msg.to}")
-#             if msg.date:
-#                 metadata.append(f"Date: {msg.date}")
+            metadata = []
 
-#             if metadata:
-#                 yield "\n".join(metadata)
+            if msg.subject:
+                metadata.append(f"Subject: {msg.subject}")
+            if msg.sender:
+                metadata.append(f"From: {msg.sender}")
+            if msg.to:
+                metadata.append(f"To: {', '.join(msg.to)}")
+            if msg.sent_date:
+                metadata.append(f"Date: {msg.sent_date}")
+            if metadata:
+                yield "\n".join(metadata)
+            if msg.body:
+                yield msg.body.strip()
 
-#             # Extract body
-#             if msg.body:
-#                 yield msg.body.strip()
+            for attachment in msg.attachments:
+                if attachment.Filename:
+                    yield f"\nAttachment: {attachment.Filename}"
 
-#             # Extract attachments (optional)
-#             for attachment in msg.attachments:
-#                 if hasattr(attachment, "name"):
-#                     yield f"\nAttachment: {attachment.name}"
-
-#         except Exception as e:
-#             raise ValueError(f"Error processing MSG file: {str(e)}")
-#         finally:
-#             file_obj.close()
+        except Exception as e:
+            raise ValueError(f"Error processing MSG file: {str(e)}") from e
+        finally:
+            os.remove(tmp_file.name)

py/core/providers/ingestion/r2r/base.py

@@ -48,7 +48,7 @@ class R2RIngestionProvider(IngestionProvider):
         DocumentType.HTM: parsers.HTMLParser,
         DocumentType.ODT: parsers.ODTParser,
         DocumentType.JSON: parsers.JSONParser,
-        # DocumentType.MSG: parsers.MSGParser,
+        DocumentType.MSG: parsers.MSGParser,
         DocumentType.ORG: parsers.ORGParser,
         DocumentType.MD: parsers.MDParser,
         DocumentType.PDF: parsers.BasicPDFParser,

py/pyproject.toml

@@ -54,6 +54,7 @@ core = [
     "hatchet-sdk ==0.47.0",
     "litellm >=1.58.2,<2.0.0",
     "markdown >=3.6,<4.0",
+    "msg-parser>=1.2.0",
     "networkx >=3.3,<4.0",
     "numpy >=1.22.4,<1.29.0",
     "olefile >=0.47,<0.48",

py/tests/integration/test_ingestion.py

@@ -161,7 +161,7 @@ def client(config):
         ("jpeg", "core/examples/supported_file_types/jpeg.jpeg"),
         ("jpg", "core/examples/supported_file_types/jpg.jpg"),
         ("md", "core/examples/supported_file_types/md.md"),
-        # ("msg", "core/examples/supported_file_types/msg.msg"),
+        ("msg", "core/examples/supported_file_types/msg.msg"),
         ("odt", "core/examples/supported_file_types/odt.odt"),
         ("org", "core/examples/supported_file_types/org.org"),
         ("p7s", "core/examples/supported_file_types/p7s.p7s"),