diff --git a/src/core/splitters/markdown.ts b/src/core/splitters/markdown.ts index 336da57..d71184c 100644 --- a/src/core/splitters/markdown.ts +++ b/src/core/splitters/markdown.ts @@ -27,6 +27,13 @@ export class MarkdownSplitter { }) } + /** + * Appends a chunk to an accumulator string, preserving whitespace + */ + reconstructChunk(accumulator: string, chunk: Chunk): string { + return accumulator + (chunk.leadingWhitespace || '') + chunk.content + (chunk.trailingWhitespace || '') + } + /** * Default split method - automatically detects and uses the appropriate splitting strategy. * If the content contains ::page directives, splits by those. diff --git a/src/core/translators/markdown.ts b/src/core/translators/markdown.ts index 8b97e6e..adf1a91 100644 --- a/src/core/translators/markdown.ts +++ b/src/core/translators/markdown.ts @@ -24,18 +24,14 @@ export class MarkdownTranslator { let response = '' for (const chunk of chunks) { - response += chunk.leadingWhitespace || '' - if (chunk.shouldTranslate) { // eslint-disable-next-line no-await-in-loop const translatedChunk = await this.chatModel.invoke(this.buildMessages({...options, content: chunk.content})) - response += translatedChunk.content as string + response = this.splitter.reconstructChunk(response, {...chunk, content: translatedChunk.content as string}) } else { - response += chunk.content + response = this.splitter.reconstructChunk(response, chunk) } - - response += chunk.trailingWhitespace || '' } return response @@ -49,9 +45,7 @@ export class MarkdownTranslator { const chunks = await this.splitter.split(options.content) for (const chunk of chunks) { - if (chunk.leadingWhitespace) { - yield chunk.leadingWhitespace - } + yield chunk.leadingWhitespace || '' if (chunk.shouldTranslate) { // eslint-disable-next-line no-await-in-loop @@ -65,9 +59,7 @@ export class MarkdownTranslator { yield chunk.content } - if (chunk.trailingWhitespace) { - yield chunk.trailingWhitespace - } + yield chunk.trailingWhitespace || '' } }