Skip to content

Commit c9f252f

Browse files
assouktimassouktim
authored
[DERCBOT-1138] PGVector (#1735)
* [DERCBOT-1138] PGVector - A new vector store * [DERCBOT-1138] PGVector - Reviews * [DERCBOT-1138] PGVector - Update dependencies
1 parent aeb71c0 commit c9f252f

File tree

67 files changed

+2792
-1571
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+2792
-1571
lines changed

‎bot/admin/server/src/main/kotlin/BotAdminVerticle.kt

+5-6
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import ai.tock.translator.I18nLabel
5757
import ai.tock.translator.Translator
5858
import ai.tock.translator.Translator.initTranslator
5959
import ai.tock.translator.TranslatorEngine
60+
import ch.tutteli.kbox.isNotNullAndNotBlank
6061
import com.fasterxml.jackson.module.kotlin.readValue
6162
import com.github.salomonbrys.kodein.instance
6263
import io.vertx.core.http.HttpMethod.GET
@@ -444,19 +445,17 @@ open class BotAdminVerticle : AdminVerticle() {
444445
}
445446
}
446447

447-
blockingJsonPost("/configuration/bots/:botId/rag", admin) { context, configuration: BotRAGConfigurationDTO ->
448-
if (context.organization == configuration.namespace) {
449-
BotRAGConfigurationDTO(RAGService.saveRag(configuration))
448+
blockingJsonPost("/configuration/bots/:botId/rag", admin) { context, request: BotRAGConfigurationDTO ->
449+
if (context.organization == request.namespace) {
450+
BotRAGConfigurationDTO(RAGService.saveRag(request))
450451
} else {
451452
unauthorized()
452453
}
453454
}
454455

455456
blockingJsonGet("/configuration/bots/:botId/rag", admin) { context ->
456457
RAGService.getRAGConfiguration(context.organization, context.path("botId"))
457-
?.let {
458-
BotRAGConfigurationDTO(it)
459-
}
458+
?.let { BotRAGConfigurationDTO(it) }
460459
}
461460

462461
blockingDelete("/configuration/bots/:botId/rag", admin) { context ->

‎bot/admin/server/src/main/kotlin/model/BotObservabilityConfigurationDTO.kt

+15-14
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import ai.tock.bot.admin.bot.observability.BotObservabilityConfiguration
2121
import ai.tock.genai.orchestratorcore.mappers.ObservabilitySettingMapper
2222
import ai.tock.genai.orchestratorcore.models.Constants
2323
import ai.tock.genai.orchestratorcore.models.observability.ObservabilitySettingDTO
24+
import ai.tock.genai.orchestratorcore.models.observability.toDTO
2425
import ai.tock.genai.orchestratorcore.utils.SecurityUtils
2526
import org.litote.kmongo.newId
2627
import org.litote.kmongo.toId
@@ -33,24 +34,24 @@ data class BotObservabilityConfigurationDTO(
3334
val setting: ObservabilitySettingDTO,
3435
) {
3536
constructor(configuration: BotObservabilityConfiguration) : this(
36-
configuration._id.toString(),
37-
configuration.namespace,
38-
configuration.botId,
39-
configuration.enabled,
40-
ObservabilitySettingMapper.toDTO(configuration.setting),
37+
id = configuration._id.toString(),
38+
namespace = configuration.namespace,
39+
botId = configuration.botId,
40+
enabled = configuration.enabled,
41+
setting = configuration.setting.toDTO(),
4142
)
4243

4344
fun toBotObservabilityConfiguration(): BotObservabilityConfiguration =
4445
BotObservabilityConfiguration(
45-
id?.toId() ?: newId(),
46-
namespace,
47-
botId,
48-
enabled,
49-
ObservabilitySettingMapper.toEntity(
50-
namespace,
51-
botId,
52-
Constants.GEN_AI_OBSERVABILITY,
53-
setting
46+
_id = id?.toId() ?: newId(),
47+
namespace = namespace,
48+
botId = botId,
49+
enabled = enabled,
50+
setting = ObservabilitySettingMapper.toEntity(
51+
namespace = namespace,
52+
botId = botId,
53+
feature = Constants.GEN_AI_OBSERVABILITY,
54+
dto = setting
5455
),
5556
)
5657
}

‎bot/admin/server/src/main/kotlin/model/BotRAGConfigurationDTO.kt

+36-24
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@
1717
package ai.tock.bot.admin.model
1818

1919
import ai.tock.bot.admin.bot.rag.BotRAGConfiguration
20+
import ai.tock.bot.admin.service.VectorStoreService
2021
import ai.tock.genai.orchestratorcore.mappers.EMSettingMapper
2122
import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
2223
import ai.tock.genai.orchestratorcore.models.Constants
2324
import ai.tock.genai.orchestratorcore.models.em.EMSettingDTO
25+
import ai.tock.genai.orchestratorcore.models.em.toDTO
2426
import ai.tock.genai.orchestratorcore.models.llm.LLMSettingDTO
25-
import ai.tock.genai.orchestratorcore.utils.SecurityUtils
27+
import ai.tock.genai.orchestratorcore.models.llm.toDTO
2628
import org.litote.kmongo.newId
2729
import org.litote.kmongo.toId
2830

@@ -34,44 +36,54 @@ data class BotRAGConfigurationDTO(
3436
val llmSetting: LLMSettingDTO,
3537
val emSetting: EMSettingDTO,
3638
val indexSessionId: String? = null,
39+
val indexName: String? = null,
3740
val noAnswerSentence: String,
3841
val noAnswerStoryId: String? = null,
3942
) {
4043
constructor(configuration: BotRAGConfiguration) : this(
41-
configuration._id.toString(),
42-
configuration.namespace,
43-
configuration.botId,
44-
configuration.enabled,
45-
LLMSettingMapper.toDTO(configuration.llmSetting),
46-
EMSettingMapper.toDTO(configuration.emSetting),
47-
configuration.indexSessionId,
48-
configuration.noAnswerSentence,
49-
configuration.noAnswerStoryId
44+
id = configuration._id.toString(),
45+
namespace = configuration.namespace,
46+
botId = configuration.botId,
47+
enabled = configuration.enabled,
48+
llmSetting = configuration.llmSetting.toDTO(),
49+
emSetting = configuration.emSetting.toDTO(),
50+
indexSessionId = configuration.indexSessionId,
51+
indexName = configuration.generateIndexName(),
52+
noAnswerSentence = configuration.noAnswerSentence,
53+
noAnswerStoryId = configuration.noAnswerStoryId
5054
)
5155

5256
fun toBotRAGConfiguration(): BotRAGConfiguration =
5357
BotRAGConfiguration(
54-
id?.toId() ?: newId(),
55-
namespace,
56-
botId,
57-
enabled,
58-
LLMSettingMapper.toEntity(
59-
namespace,
60-
botId,
61-
Constants.GEN_AI_RAG_QUESTION_ANSWERING,
62-
llmSetting
58+
_id = id?.toId() ?: newId(),
59+
namespace = namespace,
60+
botId = botId,
61+
enabled = enabled,
62+
llmSetting = LLMSettingMapper.toEntity(
63+
namespace = namespace,
64+
botId = botId,
65+
feature = Constants.GEN_AI_RAG_QUESTION_ANSWERING,
66+
dto = llmSetting
6367
),
64-
EMSettingMapper.toEntity(
65-
namespace,
66-
botId,
67-
Constants.GEN_AI_RAG_EMBEDDING_QUESTION,
68-
emSetting
68+
emSetting = EMSettingMapper.toEntity(
69+
namespace = namespace,
70+
botId = botId,
71+
feature = Constants.GEN_AI_RAG_EMBEDDING_QUESTION,
72+
dto = emSetting
6973
),
7074
indexSessionId = indexSessionId,
7175
noAnswerSentence = noAnswerSentence,
7276
noAnswerStoryId = noAnswerStoryId
7377
)
7478
}
7579

80+
private fun BotRAGConfiguration.generateIndexName(): String? {
81+
return indexSessionId?.takeIf { it.isNotBlank() }?.let {
82+
VectorStoreService.getVectorStoreConfiguration(namespace, botId, enabled = true)
83+
?.setting
84+
?.normalizeDocumentIndexName(namespace, botId, it)
85+
}
86+
}
87+
7688

7789

‎bot/admin/server/src/main/kotlin/model/BotSentenceGenerationConfigurationDTO.kt

+17-16
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import ai.tock.bot.admin.bot.sentencegeneration.BotSentenceGenerationConfigurati
2020
import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
2121
import ai.tock.genai.orchestratorcore.models.Constants
2222
import ai.tock.genai.orchestratorcore.models.llm.LLMSettingDTO
23+
import ai.tock.genai.orchestratorcore.models.llm.toDTO
2324
import ai.tock.genai.orchestratorcore.utils.SecurityUtils
2425
import org.litote.kmongo.newId
2526
import org.litote.kmongo.toId
@@ -33,26 +34,26 @@ data class BotSentenceGenerationConfigurationDTO(
3334
val llmSetting: LLMSettingDTO,
3435
) {
3536
constructor(configuration: BotSentenceGenerationConfiguration) : this(
36-
configuration._id.toString(),
37-
configuration.namespace,
38-
configuration.botId,
39-
configuration.enabled,
40-
configuration.nbSentences,
41-
LLMSettingMapper.toDTO(configuration.llmSetting),
37+
id = configuration._id.toString(),
38+
namespace = configuration.namespace,
39+
botId = configuration.botId,
40+
enabled = configuration.enabled,
41+
nbSentences = configuration.nbSentences,
42+
llmSetting = configuration.llmSetting.toDTO(),
4243
)
4344

4445
fun toSentenceGenerationConfiguration(): BotSentenceGenerationConfiguration =
4546
BotSentenceGenerationConfiguration(
46-
id?.toId() ?: newId(),
47-
namespace,
48-
botId,
49-
enabled,
50-
nbSentences,
51-
LLMSettingMapper.toEntity(
52-
namespace,
53-
botId,
54-
Constants.GEN_AI_COMPLETION_SENTENCE_GENERATION,
55-
llmSetting
47+
_id = id?.toId() ?: newId(),
48+
namespace = namespace,
49+
botId = botId,
50+
enabled = enabled,
51+
nbSentences = nbSentences,
52+
llmSetting = LLMSettingMapper.toEntity(
53+
namespace = namespace,
54+
botId = botId,
55+
feature = Constants.GEN_AI_COMPLETION_SENTENCE_GENERATION,
56+
dto = llmSetting
5657
)
5758
)
5859
}

‎bot/admin/server/src/main/kotlin/model/BotVectorStoreConfigurationDTO.kt

+15-14
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
2121
import ai.tock.genai.orchestratorcore.mappers.VectorStoreSettingMapper
2222
import ai.tock.genai.orchestratorcore.models.Constants
2323
import ai.tock.genai.orchestratorcore.models.vectorstore.VectorStoreSettingDTO
24+
import ai.tock.genai.orchestratorcore.models.vectorstore.toDTO
2425
import ai.tock.genai.orchestratorcore.utils.SecurityUtils
2526
import org.litote.kmongo.newId
2627
import org.litote.kmongo.toId
@@ -33,24 +34,24 @@ data class BotVectorStoreConfigurationDTO(
3334
val setting: VectorStoreSettingDTO,
3435
) {
3536
constructor(configuration: BotVectorStoreConfiguration) : this(
36-
configuration._id.toString(),
37-
configuration.namespace,
38-
configuration.botId,
39-
configuration.enabled,
40-
VectorStoreSettingMapper.toDTO(configuration.setting),
37+
id = configuration._id.toString(),
38+
namespace = configuration.namespace,
39+
botId = configuration.botId,
40+
enabled = configuration.enabled,
41+
setting = configuration.setting.toDTO(),
4142
)
4243

4344
fun toBotVectorStoreConfiguration(): BotVectorStoreConfiguration =
4445
BotVectorStoreConfiguration(
45-
id?.toId() ?: newId(),
46-
namespace,
47-
botId,
48-
enabled,
49-
VectorStoreSettingMapper.toEntity(
50-
namespace,
51-
botId,
52-
Constants.GEN_AI_VECTOR_STORE,
53-
setting
46+
_id = id?.toId() ?: newId(),
47+
namespace = namespace,
48+
botId = botId,
49+
enabled = enabled,
50+
setting = VectorStoreSettingMapper.toEntity(
51+
namespace = namespace,
52+
botId = botId,
53+
feature = Constants.GEN_AI_VECTOR_STORE,
54+
dto = setting
5455
)
5556
)
5657
}

‎bot/admin/server/src/main/kotlin/service/ObservabilityService.kt

+10-10
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,16 @@ object ObservabilityService {
4444
return observabilityConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
4545
}
4646

47+
/**
48+
* Get the Observability configuration
49+
* @param namespace: the namespace
50+
* @param botId: the botId
51+
* @param enabled: the observability activation (enabled or not)
52+
*/
53+
fun getObservabilityConfiguration(namespace: String, botId: String, enabled: Boolean): BotObservabilityConfiguration? {
54+
return observabilityConfigurationDAO.findByNamespaceAndBotIdAndEnabled(namespace, botId, enabled)
55+
}
56+
4757
/**
4858
* Deleting the Observability Configuration
4959
* @param namespace: the namespace
@@ -56,16 +66,6 @@ object ObservabilityService {
5666
return observabilityConfigurationDAO.delete(observabilityConfig._id)
5767
}
5868

59-
/**
60-
* Get the Observability configuration
61-
* @param namespace: the namespace
62-
* @param botId: the botId
63-
* @param enabled: the observability activation (enabled or not)
64-
*/
65-
fun getObservabilityConfiguration(namespace: String, botId: String, enabled: Boolean): BotObservabilityConfiguration? {
66-
return observabilityConfigurationDAO.findByNamespaceAndBotIdAndEnabled(namespace, botId, enabled)
67-
}
68-
6969
/**
7070
* Save Observability configuration and filter errors
7171
* @param observabilityConfig : the observability configuration to create or update

‎bot/admin/server/src/main/kotlin/service/RAGValidationService.kt

+37-19
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@ package ai.tock.bot.admin.service
1919
import ai.tock.bot.admin.bot.rag.BotRAGConfiguration
2020
import ai.tock.genai.orchestratorclient.requests.EMProviderSettingStatusQuery
2121
import ai.tock.genai.orchestratorclient.requests.LLMProviderSettingStatusQuery
22+
import ai.tock.genai.orchestratorclient.requests.VectorStoreProviderSettingStatusQuery
2223
import ai.tock.genai.orchestratorclient.responses.ProviderSettingStatusResponse
2324
import ai.tock.genai.orchestratorclient.services.EMProviderService
2425
import ai.tock.genai.orchestratorclient.services.LLMProviderService
26+
import ai.tock.genai.orchestratorclient.services.VectorStoreProviderService
27+
import ai.tock.genai.orchestratorcore.utils.VectorStoreUtils
2528
import ai.tock.shared.exception.error.ErrorMessage
2629
import ai.tock.shared.injector
2730
import ai.tock.shared.provide
@@ -31,29 +34,44 @@ object RAGValidationService {
3134

3235
private val llmProviderService: LLMProviderService get() = injector.provide()
3336
private val emProviderService: EMProviderService get() = injector.provide()
37+
private val vectorStoreProviderService: VectorStoreProviderService get() = injector.provide()
3438

3539
fun validate(ragConfig: BotRAGConfiguration): Set<ErrorMessage> {
3640
return mutableSetOf<ErrorMessage>().apply {
37-
addAll(
38-
llmProviderService
39-
.checkSetting(
40-
LLMProviderSettingStatusQuery(
41-
ragConfig.llmSetting,
42-
ObservabilityService.getObservabilityConfiguration(
43-
ragConfig.namespace,
44-
ragConfig.botId,
45-
enabled = true
46-
)?.setting
47-
)
41+
val llmErrors = llmProviderService.checkSetting(
42+
LLMProviderSettingStatusQuery(
43+
ragConfig.llmSetting,
44+
ObservabilityService.getObservabilityConfiguration(
45+
ragConfig.namespace, ragConfig.botId, enabled = true
46+
)?.setting
47+
)
48+
).getErrors("LLM setting check failed")
49+
50+
val embeddingErrors = emProviderService.checkSetting(
51+
EMProviderSettingStatusQuery(ragConfig.emSetting)
52+
).getErrors("Embedding Model setting check failed")
53+
54+
val indexSessionIdErrors = validateIndexSessionId(ragConfig)
55+
56+
val vectorStoreErrors = (indexSessionIdErrors + embeddingErrors).takeIf { it.isEmpty() }?.let {
57+
val vectorStoreSetting = VectorStoreService.getVectorStoreConfiguration(
58+
ragConfig.namespace, ragConfig.botId, enabled = true
59+
)?.setting
60+
61+
val (_, indexName) = VectorStoreUtils.getVectorStoreElements(
62+
ragConfig.namespace, ragConfig.botId, ragConfig.indexSessionId!!, vectorStoreSetting
63+
)
64+
65+
vectorStoreProviderService.checkSetting(
66+
VectorStoreProviderSettingStatusQuery(
67+
vectorStoreSetting = vectorStoreSetting,
68+
emSetting = ragConfig.emSetting,
69+
documentIndexName = indexName
4870
)
49-
.getErrors("LLM setting check failed")
50-
)
51-
addAll(
52-
emProviderService
53-
.checkSetting(EMProviderSettingStatusQuery(ragConfig.emSetting))
54-
.getErrors("Embedding Model setting check failed")
55-
)
56-
addAll(validateIndexSessionId(ragConfig))
71+
).getErrors("Vector store setting check failed")
72+
} ?: emptySet()
73+
74+
addAll(llmErrors + embeddingErrors + indexSessionIdErrors + vectorStoreErrors)
5775
}
5876
}
5977

0 commit comments

Comments
 (0)