Merge pull request #136 from gomate-community/pipeline

Pipeline

Author: yanqiangmiffy

.gitignore

@@ -31,5 +31,8 @@ examples/download/models
 .gradio
 examples/datasets/arxiv/papers
 examples/projects/arxiv/papers
-trustrag/modules/deepsearch/.env
+trustrag/modules/deepresearch/.env
 *.env
+examples/deep-research
+examples/deep-research/local-deep-research
+trustrag.egg-info

.python-version

@@ -0,0 +1 @@
+3.12

README_zh.md

@@ -376,7 +376,7 @@ app后台日志：
 >本项目感谢以下开源项目的支持与贡献：
 - 文档解析：[infiniflow/ragflow](https://github.com/infiniflow/ragflow/blob/main/deepdoc/README.md)
 - PDF文件解析：[opendatalab/MinerU](https://github.com/opendatalab/MinerU)
-- 深度搜索：[deep-research](https://github.com/dzhng/deep-research)|[deep-research-py](https://github.com/epuerta9/deep-research-py)
+- 深度搜索：[deep-research](https://github.com/dzhng/deep-research)|[deep-research-py](https://github.com/epuerta9/deep-research-py)|[deep-research](https://github.com/shibing624/deep-research/tree/main#/)
 
 ## 👉 Citation
 ```text

examples/parsers/textparser_exmaple.py

@@ -10,15 +10,18 @@
 @description: coding..
 """
 from trustrag.modules.document.txt_parser import TextParser
-
-
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.chunks.sentence_chunk import SentenceChunker
 
 
 if __name__ == '__main__':
     text_parser=TextParser()
-    chunks = text_parser.parse(fnm="../../data/docs/sample.txt")
+    tc=TextChunker()
+    sc=SentenceChunker(chunk_size=512)
+    paragraphs = text_parser.parse(fnm="../../data/docs/1737765690374-穷查理宝典.pdf-15a72b24-cc5c-4a4e-ae9e-7514e0d9be02.txt")
+    print(len(paragraphs))
+    chunks=tc.get_chunks(paragraphs,chunk_size=128)
+    # chunks=sc.get_chunks(paragraphs)
     print(len(chunks))
-
-    for chunk in chunks:
-        print("=="*100)
-        print(chunk)
+    # for chunk in chunks:
+    #     print(len(chunk))

examples/rag/rag_readme.py

@@ -0,0 +1,12 @@
+import pickle
+import pandas as pd
+from tqdm import tqdm
+
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.txt_parser import TextParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.generator.llm import GLM4Chat
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig

main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from trustrag!")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "trustrag"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = []

setup.py

@@ -18,7 +18,7 @@
     install_requires=required,
     author_email="yanqiang@ict.ac.cn",
     description="RAG Framework within Reliable input,Trusted output",
-    long_description=open('README.md').read(),
+    long_description=open('README.md','r',encoding="utf-8").read(),
     long_description_content_type='text/markdown',
     url="https://github.com/gomate-community/TrustRAG",
     python_requires='>=3.9',