chore: refactor translator core to make it easier to add more formats

Author: michaelroudnitski

src/commands/markdown.ts

@@ -1,6 +1,8 @@
 import {Args, Command, Flags} from '@oclif/core'
 
-import {MarkdownTranslator} from '../core/translators/markdown.js'
+import {MARKDOWN_SYSTEM_PROMPT} from '../core/prompts/markdown.js'
+import {MarkdownSplitter} from '../core/splitters/markdown.js'
+import {Translator} from '../core/translators/translator.js'
 import {createProviderFromProfile} from '../lib/profile/factory.js'
 import {loadProfile} from '../lib/profile/storage.js'
 
@@ -55,7 +57,7 @@ export default class Markdown extends Command {
 
     const profile = loadProfile(flags.profile)
     const llm = createProviderFromProfile(profile)
-    const translator = new MarkdownTranslator(llm)
+    const translator = new Translator(llm, new MarkdownSplitter(), MARKDOWN_SYSTEM_PROMPT)
 
     if (flags.stream) {
       for await (const chunk of translator.translateStream({

src/core/prompts/markdown.ts

@@ -0,0 +1,27 @@
+/**
+ * System prompt template for markdown translation.
+ * Uses {{sourceLanguage}} and {{targetLanguage}} as placeholders.
+ */
+export const MARKDOWN_SYSTEM_PROMPT = `
+You are a helpful assistant that accurately translates markdown document snippets from {{sourceLanguage}} to {{targetLanguage}} while preserving markdown syntax, formatting, and custom directives.
+You always preserve the structure and formatting exactly as it is.
+You do not add, alter or modify the text you receive in any way.
+
+Reminder:
+- Translate only the text, preserving the structure and formatting.
+- NEVER under any circumstances translate any words found inside backticks Eg. \`Text\`.
+- NEVER translate custom directive like ::startApplication{...} or ::openFile{...}.
+- DO translate titles inside the ::page{title=""} custom directive.
+- NEVER translate keywords that appear after colons, such as \`:fa-lightbulb-o:\`.
+- NEVER translate the sections "Author", "Other Contributors", and "Change Logs".
+- NEVER translate any URLs.
+- NEVER translate HTML tags like \`<details>\` and \`<summary>\`.
+- Translate idiomatically, adapting expressions to sound natural in {{targetLanguage}}.
+- Avoid overly literal translations; prioritize clarity and fluency in {{targetLanguage}} over word-for-word accuracy.
+- Use concise and clear language that would sound natural in everyday speech or written {{targetLanguage}}.
+- When technical {{sourceLanguage}} terms lack a common {{targetLanguage}} equivalent, use well-known {{targetLanguage}} alternatives or rephrase for clarity.
+- Be consistent with technical terms. If an equivalent technical term is not available in {{targetLanguage}}, always use the original term.
+
+*IMPORTANT*
+Translate without any additional information or comments.
+`

src/core/splitters/markdown.ts

@@ -1,6 +1,6 @@
 import {MarkdownTextSplitter} from '@langchain/textsplitters'
 
-import type {Chunk} from '../types.js'
+import type {BaseSplitter, Chunk} from '../types.js'
 
 /**
  * Hybrid two-pass markdown splitter for translation purposes.
@@ -13,7 +13,7 @@ import type {Chunk} from '../types.js'
  * - Prevents massive chunks from overwhelming translation APIs
  * - Uses LangChain's intelligent splitting for size management (tries headers > paragraphs > lines)
  */
-export class MarkdownSplitter {
+export class MarkdownSplitter implements BaseSplitter {
   private readonly chunkSize: number
   private recursiveSplitter: MarkdownTextSplitter
 
@@ -30,7 +30,7 @@ export class MarkdownSplitter {
   /**
    * Appends a chunk to an accumulator string, preserving whitespace
    */
-  reconstructChunk(accumulator: string, chunk: Chunk): string {
+  reconstruct(accumulator: string, chunk: Chunk): string {
     return accumulator + (chunk.leadingWhitespace || '') + chunk.content + (chunk.trailingWhitespace || '')
   }
 

src/core/translators/markdown.ts

@@ -0,0 +1,90 @@
+import type {BaseChatModel} from '@langchain/core/language_models/chat_models'
+
+import {BaseMessage, HumanMessage, SystemMessage} from '@langchain/core/messages'
+
+import type {BaseSplitter, TranslationOptions} from '../types.js'
+
+/**
+ * Generic translator that orchestrates content splitting, translation, and reconstruction.
+ * Works with any splitter and prompt via dependency injection.
+ */
+export class Translator {
+  constructor(
+    private chatModel: BaseChatModel,
+    private splitter: BaseSplitter,
+    private systemPromptTemplate: string,
+  ) {}
+
+  /**
+   * Translates content from source language to target language
+   */
+  async translate(options: TranslationOptions): Promise<string> {
+    const chunks = await this.splitter.split(options.content)
+    let response = ''
+
+    for (const chunk of chunks) {
+      if (chunk.shouldTranslate) {
+        // eslint-disable-next-line no-await-in-loop
+        const translatedChunk = await this.chatModel.invoke(this.buildMessages({...options, content: chunk.content}))
+
+        response = this.splitter.reconstruct(response, {...chunk, content: translatedChunk.content as string})
+      } else {
+        response = this.splitter.reconstruct(response, chunk)
+      }
+    }
+
+    return response
+  }
+
+  /**
+   * Streams translated content from source language to target language
+   * @yields {string} Chunks of translated content
+   */
+  async *translateStream(options: TranslationOptions): AsyncGenerator<string> {
+    const chunks = await this.splitter.split(options.content)
+
+    for (const chunk of chunks) {
+      yield chunk.leadingWhitespace || ''
+
+      if (chunk.shouldTranslate) {
+        // eslint-disable-next-line no-await-in-loop
+        for await (const streamedChunk of this.streamChunk({
+          ...options,
+          content: chunk.content,
+        })) {
+          yield streamedChunk
+        }
+      } else {
+        yield chunk.content
+      }
+
+      yield chunk.trailingWhitespace || ''
+    }
+  }
+
+  private buildMessages({content, sourceLanguage, targetLanguage}: TranslationOptions): BaseMessage[] {
+    const systemPrompt = this.interpolatePrompt(this.systemPromptTemplate, sourceLanguage, targetLanguage)
+    const messages = [new SystemMessage(systemPrompt), new HumanMessage(content)]
+
+    return messages
+  }
+
+  private interpolatePrompt(template: string, sourceLanguage: string, targetLanguage: string): string {
+    return template.replaceAll('{{sourceLanguage}}', sourceLanguage).replaceAll('{{targetLanguage}}', targetLanguage)
+  }
+
+  /**
+   * Streams a single chunk through the chat model
+   * @yields {string} Chunks of translated content from the model
+   */
+  private async *streamChunk(options: TranslationOptions): AsyncGenerator<string> {
+    const stream = await this.chatModel.stream(this.buildMessages(options))
+
+    for await (const chunk of stream) {
+      const {content} = chunk
+      if (typeof content === 'string') {
+        yield content
+      }
+    }
+  }
+}

src/core/translators/translator.ts

@@ -24,6 +24,22 @@ export interface Chunk {
   trailingWhitespace?: string
 }
 
+/**
+ * Base interface for content splitters.
+ * All splitter implementations must provide these methods.
+ */
+export interface BaseSplitter {
+  /**
+   * Reconstructs content by appending a chunk to an accumulator with proper whitespace
+   */
+  reconstruct(accumulator: string, chunk: Chunk): string
+
+  /**
+   * Splits content into chunks
+   */
+  split(content: string): Promise<Chunk[]>
+}
+
 /**
  * Translation configuration options
  */

src/core/types.ts

@@ -1,18 +1,20 @@
 import {describe, expect, it} from '@jest/globals'
 import {FakeListChatModel} from '@langchain/core/utils/testing'
 
-import {MarkdownTranslator} from '../../../src/core/translators/markdown.js'
+import {MARKDOWN_SYSTEM_PROMPT} from '../../../src/core/prompts/markdown.js'
+import {MarkdownSplitter} from '../../../src/core/splitters/markdown.js'
+import {Translator} from '../../../src/core/translators/translator.js'
 
 describe('MarkdownTranslator', () => {
   let fakeChatModel: FakeListChatModel
-  let translator: MarkdownTranslator
+  let translator: Translator
 
   describe('translate', () => {
     it('translates markdown content and returns string', async () => {
       fakeChatModel = new FakeListChatModel({
         responses: ['Traducido'],
       })
-      translator = new MarkdownTranslator(fakeChatModel)
+      translator = new Translator(fakeChatModel, new MarkdownSplitter(), MARKDOWN_SYSTEM_PROMPT)
 
       const result = await translator.translate({
         content: '# Hello World',
@@ -29,7 +31,7 @@ describe('MarkdownTranslator', () => {
       fakeChatModel = new FakeListChatModel({
         responses: ['Hello World'],
       })
-      translator = new MarkdownTranslator(fakeChatModel)
+      translator = new Translator(fakeChatModel, new MarkdownSplitter(), MARKDOWN_SYSTEM_PROMPT)
 
       const chunks: string[] = []
       for await (const chunk of translator.translateStream({