setup starcoderchat

Author: Anmol Singh

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 100
+per-file-ignores =
+  main.py: F401

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+__pycache__/

.pre-commit-config.yaml

@@ -0,0 +1,45 @@
+repos:
+  # Order matters because pyupgrade may make changes that pycln and black have to clean up
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.10.1
+    hooks:
+      - id: pyupgrade
+        args: [--py37-plus]
+
+  - repo: https://github.com/hadialqattan/pycln
+    rev: v2.2.0
+    hooks:
+      - id: pycln
+        args: [--config=pyproject.toml]
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+        language_version: python3.11
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-simplify
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.0
+    hooks:
+      - id: prettier

.prettierrc.yaml

@@ -0,0 +1,3 @@
+proseWrap: always
+printWidth: 88
+endOfLine: auto

README.md

@@ -0,0 +1,8 @@
+Source code for the bot hosted at: https://poe.com/StarCoderChat
+
+This is an an implementation of StarCoderChat model using the
+[Poe-Protocol](https://developer.poe.com/api-bots/poe-protocol-specification). In order
+to run this yourself, [setup modal](https://modal.com/docs/guide#getting-started), setup
+a secret on modal.com with your Poe security key and your Together API Key, rename the
+secret in main.py and then run `modal serve main.py`. The command will deploy this app
+and output a url. Use that url to create a poe bot at: https://poe.com/create_bot?api=1

main.py

@@ -0,0 +1,17 @@
+import os
+
+from fastapi_poe import make_app
+from modal import Image, Secret, Stub, asgi_app
+
+from star_coder_chat import StarCoderChatBot
+
+image = Image.debian_slim().pip_install_from_requirements("requirements.txt")
+stub = Stub("star-coder-chat-app")
+
+
+@stub.function(image=image, secret=Secret.from_name("star-coder-chat-secret"))
+@asgi_app()
+def fastapi_app():
+    bot = StarCoderChatBot(TOGETHER_API_KEY=os.environ["TOGETHER_API_KEY"])
+    app = make_app(bot, api_key=os.environ["POE_API_KEY"])
+    return app

pyproject.toml

@@ -0,0 +1,12 @@
+[tool.black]
+target-version = ['py37']
+skip-magic-trailing-comma = true
+
+[tool.pycln]
+all = true
+exclude = "main\\.py"
+
+[tool.isort]
+profile = "black"
+combine_as_imports = true
+skip_gitignore = true

requirements.txt

@@ -0,0 +1,2 @@
+fastapi-poe==0.0.14
+httpx-sse==0.3.1

star_coder_chat.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import AsyncIterable
+
+import httpx
+import httpx_sse
+from fastapi_poe import PoeBot
+from fastapi_poe.types import QueryRequest
+from sse_starlette.sse import ServerSentEvent
+
+BASE_URL = "https://api.together.xyz/inference"
+DEFAULT_SYSTEM_PROMPT = """\
+You are the StarCoderChat bot. You help users with programming and code related questions.
+Wrap any code blocks in your response in backticks so that it can be rendered using Markdown."""
+
+
+@dataclass
+class StarCoderChatBot(PoeBot):
+    TOGETHER_API_KEY: str  # Together.ai api key
+
+    def construct_prompt(self, query: QueryRequest):
+        prompt = "\n"
+        prompt += f"<system>: {DEFAULT_SYSTEM_PROMPT}\n"
+        for message in query.query:
+            if message.role == "user":
+                prompt += f"<human>: {message.content}\n"
+            elif message.role == "bot":
+                prompt += f"<bot>: {message.content}\n"
+            elif message.role == "system":
+                pass
+            else:
+                raise ValueError(f"unknown role {message.role}.")
+        prompt += "<bot>:"
+        return prompt
+
+    async def query_together_ai(self, prompt) -> str:
+        payload = {
+            "model": "HuggingFaceH4/starchat-alpha",
+            "prompt": prompt,
+            "max_tokens": 1000,
+            "stop": ["<|endoftext|>", "<|end|>", "<human>", "<bot>"],
+            "stream_tokens": True,
+            "temperature": 0.7,
+            "top_p": 0.7,
+            "top_k": 50,
+            "repetition_penalty": 1,
+        }
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "Authorization": f"Bearer {self.TOGETHER_API_KEY}",
+        }
+
+        async with httpx.AsyncClient() as aclient:
+            async with httpx_sse.aconnect_sse(
+                aclient, "POST", BASE_URL, headers=headers, json=payload
+            ) as event_source:
+                async for event in event_source.aiter_sse():
+                    if event.data != "[DONE]":
+                        token = json.loads(event.data)["choices"][0]["text"]
+                        yield token
+
+    async def get_response(self, query: QueryRequest) -> AsyncIterable[ServerSentEvent]:
+        prompt = self.construct_prompt(query)
+        async for word in self.query_together_ai(prompt):
+            yield self.text_event(word)