Skip to content

Commit bfc9f02

Browse files
phinnerphinner
committed
feat: Added bulk translation methods
1 parent 611806c commit bfc9f02

12 files changed

+200
-51
lines changed

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/BaseTranslator.kt

+5-2
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,12 @@ internal abstract class BaseTranslator : Translator {
3333
override fun translate(text: String, source: Locale, target: Locale): CompletableFuture<String> =
3434
translateDetecting(text, source, target).thenApply(TranslatedText::text)
3535

36+
override fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> =
37+
translateDetecting(listOf(text), source, target).thenApply(List<TranslatedText>::first)
38+
3639
abstract override fun translateDetecting(
37-
text: String,
40+
texts: List<String>,
3841
source: Locale,
3942
target: Locale,
40-
): CompletableFuture<TranslatedText>
43+
): CompletableFuture<List<TranslatedText>>
4144
}

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/CachingTranslator.kt

+44
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,56 @@ internal constructor(
6464
result.fold({ CompletableFuture.completedFuture(it) }, { CompletableFuture.failedFuture(it) })
6565
}
6666

67+
override fun translateDetecting(
68+
texts: List<String>,
69+
source: Locale,
70+
target: Locale,
71+
): CompletableFuture<List<TranslatedText>> {
72+
val sourceLanguage = Locale.forLanguageTag(source.language)
73+
val targetLanguage = Locale.forLanguageTag(target.language)
74+
val keys = texts.map { TranslationKey(it, sourceLanguage, targetLanguage) }
75+
return cache.getAll(keys).thenCompose { map ->
76+
val list = ArrayList<TranslatedText>(keys.size)
77+
for (i in keys.indices) {
78+
val result =
79+
map[keys[i]]
80+
?: return@thenCompose CompletableFuture.failedFuture(
81+
NullPointerException(
82+
"Missing key ${keys[i]} in results for text $texts translating from $source to $target"
83+
)
84+
)
85+
list.add(
86+
result.getOrNull() ?: return@thenCompose CompletableFuture.failedFuture(result.exceptionOrNull()!!)
87+
)
88+
}
89+
CompletableFuture.completedFuture(list)
90+
}
91+
}
92+
6793
private inner class TranslationLoader : AsyncCacheLoader<TranslationKey, Result<TranslatedText>> {
6894
override fun asyncLoad(key: TranslationKey, executor: Executor): CompletableFuture<Result<TranslatedText>> =
6995
translator
7096
.translateDetecting(key.text, key.source, key.target)
7197
.thenApply<Result<TranslatedText>> { Result.success(it) }
7298
.exceptionally { Result.failure(it) }
99+
100+
override fun asyncLoadAll(
101+
keys: Set<TranslationKey>,
102+
executor: Executor,
103+
): CompletableFuture<out Map<TranslationKey, Result<TranslatedText>>> {
104+
val groups =
105+
keys
106+
.groupBy { it.source to it.target }
107+
.map { (pair, keys) ->
108+
val (source, target) = pair
109+
translator.translateDetecting(keys.map(TranslationKey::text), source, target).thenApply {
110+
it.mapIndexed { i, t -> keys[i] to Result.success(t) }
111+
}
112+
}
113+
return CompletableFuture.allOf(*groups.toTypedArray()).thenApply { _ ->
114+
groups.map { it.join() }.flatten().toMap()
115+
}
116+
}
73117
}
74118

75119
private inner class TranslationExpiry : Expiry<TranslationKey, Result<TranslatedText>> {

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/DeepLTranslator.kt

+17-14
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,14 @@ internal class DeepLTranslator(apiKey: String, private val executor: Executor) :
4040
internal val sourceLanguages: List<Locale> = fetchLanguages(LanguageType.Source)
4141
internal val targetLanguages: List<Locale> = fetchLanguages(LanguageType.Target)
4242

43-
override fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> {
44-
if (text.isBlank()) {
45-
return CompletableFuture.completedFuture(TranslatedText(text))
46-
} else if (source == Translator.ROUTER || target == Translator.ROUTER) {
47-
return CompletableFuture.completedFuture(TranslatedText("router"))
43+
override fun translateDetecting(
44+
texts: List<String>,
45+
source: Locale,
46+
target: Locale,
47+
): CompletableFuture<List<TranslatedText>> {
48+
if (source == Translator.ROUTER || target == Translator.ROUTER) {
49+
val result = TranslatedText("router")
50+
return CompletableFuture.completedFuture(List(texts.size) { result })
4851
}
4952

5053
val sourceLocale =
@@ -60,7 +63,7 @@ internal class DeepLTranslator(apiKey: String, private val executor: Executor) :
6063
?: return CompletableFuture.failedFuture(UnsupportedLanguageException(target))
6164

6265
if (sourceLocale?.language == targetLocale.language) {
63-
return CompletableFuture.completedFuture(TranslatedText(text, sourceLocale))
66+
return CompletableFuture.completedFuture(List(texts.size) { i -> TranslatedText(texts[i], sourceLocale) })
6467
}
6568

6669
return CompletableFuture.runAsync(
@@ -72,14 +75,14 @@ internal class DeepLTranslator(apiKey: String, private val executor: Executor) :
7275
executor,
7376
)
7477
.thenApply {
75-
val result =
76-
translator.translateText(
77-
text,
78-
sourceLocale?.language,
79-
targetLocale.toLanguageTag(),
80-
DEFAULT_OPTIONS,
81-
)
82-
TranslatedText(result.text, sourceLocale ?: Locale.forLanguageTag(result.detectedSourceLanguage))
78+
translator
79+
.translateText(texts, sourceLocale?.language, targetLocale.toLanguageTag(), DEFAULT_OPTIONS)
80+
.map { result ->
81+
TranslatedText(
82+
result.text,
83+
sourceLocale ?: Locale.forLanguageTag(result.detectedSourceLanguage),
84+
)
85+
}
8386
}
8487
}
8588

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/GoogleBasicTranslator.kt

+31-17
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ import java.util.Locale
3333
import java.util.concurrent.CompletableFuture
3434
import java.util.concurrent.Executor
3535
import kotlinx.serialization.json.Json
36+
import kotlinx.serialization.json.JsonArray
37+
import kotlinx.serialization.json.JsonPrimitive
3638
import kotlinx.serialization.json.jsonArray
3739
import kotlinx.serialization.json.jsonObject
3840
import kotlinx.serialization.json.jsonPrimitive
@@ -42,11 +44,14 @@ internal class GoogleBasicTranslator(private val apiKey: String, executor: Execu
4244
private val http = HttpClient.newBuilder().executor(executor).build()
4345
internal val supported: Set<Locale> = fetchSupportedLanguages()
4446

45-
override fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> {
47+
override fun translateDetecting(
48+
texts: List<String>,
49+
source: Locale,
50+
target: Locale,
51+
): CompletableFuture<List<TranslatedText>> {
4652
if (source == Translator.ROUTER || target == Translator.ROUTER) {
47-
return CompletableFuture.completedFuture(TranslatedText("router"))
48-
} else if (text.isBlank() || source.language == target.language) {
49-
return CompletableFuture.completedFuture(TranslatedText(text, source))
53+
val result = TranslatedText("router")
54+
return CompletableFuture.completedFuture(List(texts.size) { result })
5055
}
5156

5257
var fixedSource = source
@@ -63,8 +68,17 @@ internal class GoogleBasicTranslator(private val apiKey: String, executor: Execu
6368
}
6469
}
6570

71+
if (fixedSource.language == fixedTarget.language) {
72+
return CompletableFuture.completedFuture(List(texts.size) { i -> TranslatedText(texts[i], fixedSource) })
73+
}
74+
6675
val parameters =
67-
mutableMapOf("key" to apiKey, "q" to text, "target" to fixedTarget.toLanguageTag(), "format" to "text")
76+
mutableMapOf(
77+
"key" to apiKey,
78+
"q" to JsonArray(texts.map(::JsonPrimitive)).toString(),
79+
"target" to fixedTarget.toLanguageTag(),
80+
"format" to "text",
81+
)
6882

6983
if (fixedSource != Translator.AUTO_DETECT) {
7084
parameters["source"] = fixedSource.toLanguageTag()
@@ -79,22 +93,22 @@ internal class GoogleBasicTranslator(private val apiKey: String, executor: Execu
7993
if (response.statusCode() != 200) {
8094
CompletableFuture.failedFuture(Exception("Failed to translate text: ${response.statusCode()}"))
8195
} else {
82-
val result =
96+
CompletableFuture.completedFuture(
8397
Json.parseToJsonElement(response.body())
8498
.jsonObject["data"]!!
8599
.jsonObject["translations"]!!
86100
.jsonArray
87-
.first()
88-
.jsonObject
89-
CompletableFuture.completedFuture(
90-
TranslatedText(
91-
result["translatedText"]!!.jsonPrimitive.content,
92-
if (fixedSource == Translator.AUTO_DETECT) {
93-
Locale.forLanguageTag(result["detectedSourceLanguage"]!!.jsonPrimitive.content)
94-
} else {
95-
fixedSource
96-
},
97-
)
101+
.map { element ->
102+
val obj = element.jsonObject
103+
TranslatedText(
104+
obj["translatedText"]!!.jsonPrimitive.content,
105+
if (fixedSource == Translator.AUTO_DETECT) {
106+
Locale.forLanguageTag(obj["detectedSourceLanguage"]!!.jsonPrimitive.content)
107+
} else {
108+
fixedSource
109+
},
110+
)
111+
}
98112
)
99113
}
100114
}

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/LibreTranslateTranslator.kt

+11
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@ constructor(private val endpoint: URI, executor: Executor, private val apiKey: S
4646
private val http = HttpClient.newBuilder().executor(executor).build()
4747
internal val languages: Map<String, Set<String>> = fetchSupportedLanguages()
4848

49+
override fun translateDetecting(
50+
texts: List<String>,
51+
source: Locale,
52+
target: Locale,
53+
): CompletableFuture<List<TranslatedText>> {
54+
val futures = texts.map { text -> translateDetecting(text, source, target) }
55+
return CompletableFuture.allOf(*futures.toTypedArray()).thenApply {
56+
futures.map(CompletableFuture<TranslatedText>::join)
57+
}
58+
}
59+
4960
override fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> {
5061
if (source == Translator.ROUTER || target == Translator.ROUTER) {
5162
return CompletableFuture.completedFuture(TranslatedText("router"))

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/RollingTranslator.kt

+12-18
Original file line numberDiff line numberDiff line change
@@ -33,27 +33,21 @@ internal class RollingTranslator(private val translators: List<Translator>, priv
3333
BaseTranslator() {
3434
private val cursor = AtomicInteger(0)
3535

36-
override fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> {
37-
val cursor = cursor.getAndUpdate { if (it + 1 < translators.size) it + 1 else 0 }
38-
return translate0(text, source, target, cursor, 0)
36+
override fun translateDetecting(text: String, source: Locale, target: Locale) = roll {
37+
it.translateDetecting(text, source, target)
3938
}
4039

41-
private fun translate0(
42-
text: String,
43-
source: Locale,
44-
target: Locale,
45-
cursor: Int,
46-
index: Int,
47-
): CompletableFuture<TranslatedText> {
48-
if (index >= translators.size) return fallback.translateDetecting(text, source, target)
49-
val translator = translators[(cursor + index) % translators.size]
50-
return translator.translateDetecting(text, source, target).exceptionallyCompose { throwable ->
51-
logger.log(System.Logger.Level.DEBUG, "Translator {0} failed", translator.javaClass.simpleName, throwable)
52-
translate0(text, source, target, cursor, index + 1)
53-
}
40+
override fun translateDetecting(texts: List<String>, source: Locale, target: Locale) = roll {
41+
it.translateDetecting(texts, source, target)
5442
}
5543

56-
private companion object {
57-
@JvmStatic private val logger = System.getLogger(RollingTranslator::class.java.name)
44+
private fun <T : Any> roll(
45+
idx: Int = 0,
46+
cur: Int = cursor.getAndUpdate { if (it + 1 < translators.size) it + 1 else 0 },
47+
function: (Translator) -> CompletableFuture<T>,
48+
): CompletableFuture<T> {
49+
if (idx >= translators.size) return function(fallback)
50+
val translator = translators[(cur + idx) % translators.size]
51+
return function(translator).exceptionallyCompose { roll(cur, idx + 1, function) }
5852
}
5953
}

‎flex-translator/src/main/kotlin/com/xpdustry/flex/translator/Translator.kt

+11
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@ public interface Translator {
3939
public fun translateDetecting(text: String, source: Locale, target: Locale): CompletableFuture<TranslatedText> =
4040
translate(text, source, target).thenApply { TranslatedText(it, target) }
4141

42+
public fun translateDetecting(
43+
texts: List<String>,
44+
source: Locale,
45+
target: Locale,
46+
): CompletableFuture<List<TranslatedText>> {
47+
val futures = texts.map { translateDetecting(it, source, target) }
48+
return CompletableFuture.allOf(*futures.toTypedArray()).thenApply { _ ->
49+
futures.map { future -> future.join() }
50+
}
51+
}
52+
4253
@Deprecated("Deprecated", ReplaceWith("Translator.noop()"))
4354
public object None : Translator {
4455
@Deprecated("Deprecated", ReplaceWith("translateDetecting(text, source, target)"))

‎flex-translator/src/test/kotlin/com/xpdustry/flex/translator/CachingTranslatorTest.kt

+35
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,41 @@ class CachingTranslatorTest {
124124
Assertions.assertEquals(2, translator.successCount)
125125
}
126126

127+
@Test
128+
fun `test multiple`() {
129+
val translator = TestTranslator()
130+
131+
val key1 = TranslationKey("hello", Locale.ENGLISH, Locale.FRENCH)
132+
val val1 = translationSuccess("bonjour")
133+
val key2 = TranslationKey("hi", Locale.ENGLISH, Locale.FRENCH)
134+
val val2 = translationSuccess("salut")
135+
val key3 = TranslationKey("weird", Locale.ENGLISH, Locale.FRENCH)
136+
val val3 = translationSuccess("bizarre")
137+
138+
translator.results[key1] = val1
139+
translator.results[key2] = val2
140+
translator.results[key3] = val3
141+
142+
val ticker = NavigableTicker()
143+
val caching =
144+
CachingTranslator(
145+
translator,
146+
Runnable::run,
147+
1000,
148+
5.minutes.toJavaDuration(),
149+
5.seconds.toJavaDuration(),
150+
ticker,
151+
)
152+
153+
Assertions.assertEquals(
154+
listOf(val1.getOrThrow(), val2.getOrThrow(), val3.getOrThrow()),
155+
caching.translateDetecting(listOf(key1.text, key2.text, key3.text), key1.source, key1.target).join(),
156+
)
157+
158+
Assertions.assertEquals(1, translator.successCount)
159+
Assertions.assertEquals(0, translator.failureCount)
160+
}
161+
127162
private fun assertThrowsCompletable(exception: KClass<out Throwable>, future: CompletableFuture<*>) {
128163
try {
129164
future.join()

‎flex-translator/src/test/kotlin/com/xpdustry/flex/translator/DeepLTranslatorTest.kt

+4
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ class DeepLTranslatorTest {
4545
val result = translator.translateDetecting("Bonjour", Translator.AUTO_DETECT, Locale.ENGLISH).join()
4646
Assertions.assertEquals(Locale.FRENCH.language, result.detected?.language)
4747
}
48+
Assertions.assertDoesNotThrow {
49+
val result = translator.translateDetecting(listOf("Bonjour", "Salut"), Locale.FRENCH, Locale.ENGLISH).join()
50+
Assertions.assertEquals(2, result.size)
51+
}
4852
}
4953

5054
companion object {

‎flex-translator/src/test/kotlin/com/xpdustry/flex/translator/GoogleBasicTranslatorTest.kt

+4
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ class GoogleBasicTranslatorTest {
4646
val result = translator.translateDetecting("Bonjour", Translator.AUTO_DETECT, Locale.ENGLISH).join()
4747
Assertions.assertEquals(Locale.FRENCH.language, result.detected?.language)
4848
}
49+
Assertions.assertDoesNotThrow {
50+
val result = translator.translateDetecting(listOf("Bonjour", "Salut"), Locale.FRENCH, Locale.ENGLISH).join()
51+
Assertions.assertEquals(2, result.size)
52+
}
4953
}
5054

5155
companion object {

‎flex-translator/src/test/kotlin/com/xpdustry/flex/translator/LibreTranslateTranslatorTest.kt

+4
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ class LibreTranslateTranslatorTest {
4242
Assertions.assertTrue(translator.languages.isNotEmpty())
4343
Assertions.assertTrue(translator.languages.values.flatten().isNotEmpty())
4444
Assertions.assertDoesNotThrow { translator.translateDetecting("Bonjour", Locale.FRENCH, Locale.ENGLISH).join() }
45+
Assertions.assertDoesNotThrow {
46+
val result = translator.translateDetecting(listOf("Bonjour", "Salut"), Locale.FRENCH, Locale.ENGLISH).join()
47+
Assertions.assertEquals(2, result.size)
48+
}
4549
}
4650

4751
companion object {

‎flex-translator/src/test/kotlin/com/xpdustry/flex/translator/TestTranslator.kt

+22
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,26 @@ internal class TestTranslator : BaseTranslator() {
4848
CompletableFuture.failedFuture(it)
4949
},
5050
) ?: CompletableFuture.failedFuture(UnsupportedOperationException("No result for $text"))
51+
52+
override fun translateDetecting(
53+
texts: List<String>,
54+
source: Locale,
55+
target: Locale,
56+
): CompletableFuture<List<TranslatedText>> {
57+
val values =
58+
texts.map { text ->
59+
results[TranslationKey(text, source, target)]
60+
?: run {
61+
failureCount++
62+
return CompletableFuture.failedFuture(UnsupportedOperationException("No result for $text"))
63+
}
64+
}
65+
return CompletableFuture.completedFuture(
66+
values.map { translated ->
67+
if (translated.isFailure) return CompletableFuture.failedFuture(translated.exceptionOrNull()!!)
68+
translated.getOrNull()!!
69+
}
70+
)
71+
.whenComplete { _, throwable -> if (throwable == null) successCount++ }
72+
}
5173
}

0 commit comments

Comments
 (0)