diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImpl.java index 2a8d06490..fd0cf761a 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImpl.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImpl.java @@ -47,6 +47,7 @@ import org.thingsboard.mqtt.broker.service.security.authorization.AuthRulePatterns; import org.thingsboard.mqtt.broker.session.ClientMqttActorManager; import org.thingsboard.mqtt.broker.session.ClientSessionCtx; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.session.DisconnectReason; import org.thingsboard.mqtt.broker.session.DisconnectReasonType; import org.thingsboard.mqtt.broker.util.MqttReasonCodeResolver; @@ -81,6 +82,7 @@ public class ActorProcessorImpl implements ActorProcessor { private final UnauthorizedClientManager unauthorizedClientManager; private final BlockedClientService blockedClientService; private final AuthorizationRoutingService authorizationRoutingService; + private final IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; @Override public void onInit(ClientActorState state, SessionInitMsg sessionInitMsg) { @@ -96,11 +98,13 @@ public void onInit(ClientActorState state, SessionInitMsg sessionInitMsg) { } AuthContext authContext = buildAuthContext(state, sessionInitMsg); + sessionCtx.setUsername(authContext.getUsername()); AuthResponse authResponse = authorizationRoutingService.executeAuthFlow(authContext); if (authResponse.notSuccess()) { log.warn("[{}] Connection is not established due to: {}", state.getClientId(), CONNECTION_REFUSED_NOT_AUTHORIZED); unauthorizedClientManager.persistClientUnauthorized(state, sessionInitMsg, authResponse.getReason()); + integrationLifecycleEventPublisher.publishAuthenticationFailed(sessionCtx, state.getClientId(), authResponse.getReason()); sendConnectionRefusedNotAuthorizedMsgAndCloseChannel(sessionCtx); return; } @@ -203,10 +207,14 @@ public void onEnhancedReAuth(ClientActorState state, MqttAuthMsg authMsg) { private void processAuth(ClientActorState state, MqttAuthMsg authMsg, ClientSessionCtx sessionCtx) { EnhancedAuthContext authContext = buildEnhancedAuthContext(state, authMsg); EnhancedAuthFinalResponse authResponse = enhancedAuthenticationService.onAuthContinue(sessionCtx, authContext); + sessionCtx.setUsername(authResponse.username()); if (!authResponse.success()) { resetStateToDisconnected(state); MqttConnectReturnCode returnCode = getFailureReturnCode(authResponse); unauthorizedClientManager.persistClientUnauthorized(state, sessionCtx, authResponse); + // Carry the enhanced-auth failure cause (e.g. AUTH_METHOD_MISMATCH) so CLIENT_AUTHENTICATION_FAILED.reason + // describes why authentication failed on both paths, rather than leaking the MQTT CONNACK return-code name here. + integrationLifecycleEventPublisher.publishAuthenticationFailed(sessionCtx, state.getClientId(), authResponse.enhancedAuthFailure().name()); sendConnectionRefusedMsgAndCloseChannel(sessionCtx, returnCode); return; } diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializer.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializer.java index 358816499..36607e0c6 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializer.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializer.java @@ -15,6 +15,7 @@ */ package org.thingsboard.mqtt.broker.actors.client.service; +import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -24,11 +25,15 @@ import org.thingsboard.mqtt.broker.actors.client.service.session.ClientSessionService; import org.thingsboard.mqtt.broker.actors.client.service.subscription.ClientSubscriptionService; import org.thingsboard.mqtt.broker.common.data.ClientSessionInfo; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; +import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; import org.thingsboard.mqtt.broker.config.ClientsLimitProperties; import org.thingsboard.mqtt.broker.dao.integration.IntegrationService; import org.thingsboard.mqtt.broker.exception.QueuePersistenceException; import org.thingsboard.mqtt.broker.queue.cluster.ServiceInfoProvider; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventTypeCache; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.client.blocked.BlockedClientService; import org.thingsboard.mqtt.broker.service.mqtt.client.blocked.consumer.BlockedClientConsumerService; @@ -50,6 +55,7 @@ import org.thingsboard.mqtt.broker.service.subscription.data.SubscriptionsSourceKey; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -74,6 +80,7 @@ public class BrokerInitializer { private final ServiceInfoProvider serviceInfoProvider; private final RateLimitService rateLimitService; private final IntegrationService integrationService; + private final IntegrationLifecycleEventTypeCache lifecycleEventTypeCache; private final ClientsLimitProperties clientsLimitProperties; private final ClientSessionEventConsumer clientSessionEventConsumer; @@ -89,6 +96,7 @@ public class BrokerInitializer { public void onApplicationEvent(ApplicationReadyEvent event) { log.info("Initializing Client Sessions and Subscriptions."); try { + initIntegrationLifecycleEventCache(); Map allClientSessions = initClientSessions(); initClientSubscriptions(allClientSessions); @@ -109,6 +117,25 @@ public void onApplicationEvent(ApplicationReadyEvent event) { } } + void initIntegrationLifecycleEventCache() { + List integrations = integrationService.findAllIntegrations(); + int cached = 0; + for (Integration integration : integrations) { + JsonNode configuration = integration.getConfiguration(); + if (configuration == null || !configuration.has(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY)) { + continue; + } + Set eventTypes = ClientLifecycleEventTypeUtil.parse( + configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY), + name -> log.warn("[{}] Unknown lifecycle event type: {}", integration.getId(), name)); + if (!eventTypes.isEmpty()) { + lifecycleEventTypeCache.put(integration.getIdStr(), eventTypes); + cached++; + } + } + log.info("Loaded lifecycle event type cache: cached {} of {} integrations.", cached, integrations.size()); + } + Map initClientSessions() throws QueuePersistenceException { Map allClientSessions = clientSessionConsumer.initLoad(); log.info("Loaded {} stored client sessions from Kafka.", allClientSessions.size()); diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImpl.java index 87fffe351..993bfb24e 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImpl.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImpl.java @@ -57,6 +57,7 @@ import org.thingsboard.mqtt.broker.service.mqtt.persistence.MsgPersistenceManager; import org.thingsboard.mqtt.broker.service.mqtt.validation.PublishMsgValidationService; import org.thingsboard.mqtt.broker.service.mqtt.will.LastWillService; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.stats.StatsManager; import org.thingsboard.mqtt.broker.service.subscription.ClientSubscriptionCache; import org.thingsboard.mqtt.broker.service.subscription.shared.TopicSharedSubscription; @@ -94,6 +95,7 @@ public class ConnectServiceImpl implements ConnectService { private final PublishMsgValidationService publishMsgValidationService; private final MqttPublishMsgDeliveryService mqttPublishMsgDeliveryService; private final StatsManager statsManager; + private final IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; private ExecutorService connectHandlerExecutor; @@ -209,6 +211,7 @@ public void acceptConnection(ClientActorStateInfo actorState, ConnectionAccepted log.debug("[{}] [{}] Client connected!", actorState.getClientId(), actorState.getCurrentSessionId()); clientSessionCtxService.registerSession(sessionCtx); + integrationLifecycleEventPublisher.publishConnected(sessionCtx); if (sessionCtx.getSessionInfo().isPersistent()) { msgPersistenceManager.startProcessingPersistedMessages(actorState); @@ -243,12 +246,16 @@ private void pushConnAckMsg(ClientActorStateInfo actorState, ConnectionAcceptedM void refuseConnection(ClientSessionCtx clientSessionCtx, ClientSessionFailureReason reason, Throwable t) { logConnectionRefused(clientSessionCtx, reason, t); - sendConnectionRefusedMsgAndDisconnect(clientSessionCtx, reason); + MqttConnectReturnCode returnCode = reason.toMqttReturnCode(clientSessionCtx); + // Emit the same MQTT CONNACK reason-code name the client receives, matching the pre-connection validation + // path (which emits MqttConnectReturnCode.name()) so CLIENT_CONNECTION_FAILED speaks a single vocabulary. + integrationLifecycleEventPublisher.publishConnectionFailed(clientSessionCtx, clientSessionCtx.getSessionInfo(), returnCode.name()); + + sendConnectionRefusedMsgAndDisconnect(clientSessionCtx, returnCode); } - private void sendConnectionRefusedMsgAndDisconnect(ClientSessionCtx ctx, ClientSessionFailureReason reason) { + private void sendConnectionRefusedMsgAndDisconnect(ClientSessionCtx ctx, MqttConnectReturnCode mqttReturnCode) { try { - MqttConnectReturnCode mqttReturnCode = reason.toMqttReturnCode(ctx); createAndSendConnAckMsg(mqttReturnCode, ctx); } catch (Exception e) { log.warn("[{}][{}] Failed to send CONN_ACK response.", ctx.getClientId(), ctx.getSessionId()); @@ -295,6 +302,9 @@ boolean shouldProceedWithConnection(ClientActorStateInfo actorState, MqttConnect validateLastWillMessage(ctx, clientId, msg); } catch (ConnectionValidationException e) { log.warn("[{}] Connection validation failed: {}", ctx.getSessionId(), e.getMessage()); + // ctx.getSessionInfo() is not set yet at this point, so pass the already-built sessionInfo explicitly. + // reason is the MQTT connect return code the client received, e.g. CONNECTION_REFUSED_CLIENT_IDENTIFIER_NOT_VALID. + integrationLifecycleEventPublisher.publishConnectionFailed(ctx, sessionInfo, e.getMqttConnectReturnCode().name()); createAndSendConnAckMsg(e.getMqttConnectReturnCode(), ctx); disconnect(clientId, ctx.getSessionId()); return false; diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImpl.java index 03c1a1592..d00ba270c 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImpl.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImpl.java @@ -27,6 +27,7 @@ import org.thingsboard.mqtt.broker.common.data.ClientInfo; import org.thingsboard.mqtt.broker.common.data.SessionInfo; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRuleService; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.historical.stats.TbMessageStatsReportClient; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; @@ -60,6 +61,7 @@ public class DisconnectServiceImpl implements DisconnectService { private final FlowControlService flowControlService; private final TbMessageStatsReportClient tbMessageStatsReportClient; private final ChannelBackpressureManager channelBackpressureManager; + private final IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; @Override public void disconnect(ClientActorStateInfo actorState, MqttDisconnectMsg disconnectMsg) { @@ -85,6 +87,15 @@ public void disconnect(ClientActorStateInfo actorState, MqttDisconnectMsg discon if (reasonType.isNotClusterConflictingSession()) { notifyClientDisconnected(actorState, sessionExpiryInterval, reasonType); } + // Emit the lifecycle CLIENT_DISCONNECTED on every disconnect, including cross-node session takeover + // (ON_CLUSTER_CONFLICTING_SESSIONS). The session-event notification above is intentionally suppressed + // on takeover, but the lifecycle event must still fire to pair with the CLIENT_CONNECTED this node emitted. + // Exception: a broker-refused connection (ON_CONNECTION_FAILURE) never established a session and never + // emitted CLIENT_CONNECTED, so its teardown must not emit a phantom CLIENT_DISCONNECTED; the dedicated + // CLIENT_CONNECTION_FAILED event covers that case instead. + if (reasonType != DisconnectReasonType.ON_CONNECTION_FAILURE) { + integrationLifecycleEventPublisher.publishDisconnected(sessionCtx, reasonType); + } cleanupClientSession(actorState, disconnectMsg, sessionExpiryInterval); } diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandler.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandler.java index d186c0473..a414a4b80 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandler.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandler.java @@ -34,6 +34,8 @@ import org.thingsboard.mqtt.broker.dao.topic.TopicValidationService; import org.thingsboard.mqtt.broker.exception.DataValidationException; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRuleService; +import org.thingsboard.mqtt.broker.service.integration.AuthorizationAction; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.MqttMsgDeliveryService; @@ -74,6 +76,7 @@ public class MqttSubscribeHandler { private final MsgPersistenceManager msgPersistenceManager; private final ApplicationPersistenceProcessor applicationPersistenceProcessor; private final RateLimitService rateLimitService; + private final IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; public void process(ClientSessionCtx ctx, MqttSubscribeMsg msg) { Set currentSharedSubscriptions = clientSubscriptionService.getClientSharedSubscriptions(ctx.getClientId()); @@ -132,6 +135,7 @@ List collectMqttReasonCodes(ClientSessionCtx ctx, MqttSu if (!isClientAuthorized) { log.warn("[{}][{}] Client is not authorized to subscribe to the topic {}", ctx.getClientId(), ctx.getSessionId(), topic); + integrationLifecycleEventPublisher.publishAuthorizationDenied(ctx, AuthorizationAction.SUBSCRIBE, topic); codes.add(MqttReasonCodeResolver.notAuthorizedSubscribe(ctx)); continue; } @@ -162,6 +166,7 @@ private void subscribeAndPersist(ClientSessionCtx ctx, List n CallbackUtil.createCallback( () -> { sendSubAck(ctx, subAckMessage); + integrationLifecycleEventPublisher.publishSubscribed(ctx, newSubscriptions); processRetainedMessages(ctx, newSubscriptions, currentSubscriptions); }, t -> log.warn("[{}][{}] Failed to process client subscription.", clientId, ctx.getSessionId(), t)) diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandler.java b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandler.java index 11df04d07..f5fc8883d 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandler.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandler.java @@ -24,14 +24,19 @@ import org.thingsboard.mqtt.broker.actors.client.messages.mqtt.MqttUnsubscribeMsg; import org.thingsboard.mqtt.broker.actors.client.service.subscription.ClientSubscriptionService; import org.thingsboard.mqtt.broker.adaptor.NettyMqttConverter; +import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; import org.thingsboard.mqtt.broker.common.data.util.CallbackUtil; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.persistence.application.ApplicationPersistenceProcessor; import org.thingsboard.mqtt.broker.service.subscription.shared.TopicSharedSubscription; import org.thingsboard.mqtt.broker.session.ClientSessionCtx; import org.thingsboard.mqtt.broker.util.MqttReasonCodeResolver; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -43,20 +48,59 @@ public class MqttUnsubscribeHandler { private final MqttMessageGenerator mqttMessageGenerator; private final ClientSubscriptionService clientSubscriptionService; private final ApplicationPersistenceProcessor applicationPersistenceProcessor; + private final IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; public void process(ClientSessionCtx ctx, MqttUnsubscribeMsg msg) { log.trace("[{}][{}] Processing unsubscribe, messageId - {}, topic filters - {}", ctx.getClientId(), ctx.getSessionId(), msg.getMessageId(), msg.getTopics()); MqttMessage unSubAckMessage = mqttMessageGenerator.createUnSubAckMessage(msg.getMessageId(), getCodes(ctx, msg)); + // MQTT allows UNSUBSCRIBE for filters the client never subscribed to. Emit CLIENT_UNSUBSCRIBED only for + // the subscriptions actually removed (symmetric with CLIENT_SUBSCRIBED, which emits only the granted ones). + List removedSubscriptions = getRemovedSubscriptions(ctx.getClientId(), msg.getTopics()); clientSubscriptionService.unsubscribeAndPersist(ctx.getClientId(), msg.getTopics(), CallbackUtil.createCallback( - () -> ctx.getChannel().writeAndFlush(unSubAckMessage), + () -> { + ctx.getChannel().writeAndFlush(unSubAckMessage); + if (!removedSubscriptions.isEmpty()) { + integrationLifecycleEventPublisher.publishUnsubscribed(ctx, removedSubscriptions); + } + }, t -> log.warn("[{}][{}] Failed to process client unsubscription", ctx.getClientId(), ctx.getSessionId(), t) )); stopProcessingApplicationSharedSubscriptions(ctx, msg.getTopics()); } + private List getRemovedSubscriptions(String clientId, List requestedTopics) { + Set currentSubscriptions = clientSubscriptionService.getClientSubscriptions(clientId); + if (CollectionUtils.isEmpty(currentSubscriptions)) { + return List.of(); + } + // Index current subscriptions by (shareName, bare topic filter) so a requested $share// + // resolves to that exact shared subscription (carrying its shareName), and a bare filter to the regular one. + Map byKey = new HashMap<>(); + for (TopicSubscription sub : currentSubscriptions) { + byKey.putIfAbsent(new SubKey(sub.getShareName(), sub.getTopicFilter()), sub); + } + List removed = new ArrayList<>(); + for (String requested : requestedTopics) { + String shareName = NettyMqttConverter.isSharedTopic(requested) ? NettyMqttConverter.getShareName(requested) : null; + TopicSubscription sub = byKey.get(new SubKey(shareName, toTopicFilter(requested))); + if (sub != null) { + removed.add(sub); + } + } + return removed; + } + + // Composite lookup key; shareName is null for a regular subscription, the group name for a shared one. + private record SubKey(String shareName, String topicFilter) { + } + + private static String toTopicFilter(String topic) { + return NettyMqttConverter.isSharedTopic(topic) ? NettyMqttConverter.getTopicFilter(topic) : topic; + } + private List getCodes(ClientSessionCtx ctx, MqttUnsubscribeMsg msg) { return msg .getTopics() diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/adaptor/ProtoConverter.java b/application/src/main/java/org/thingsboard/mqtt/broker/adaptor/ProtoConverter.java index 382b48f99..33890830a 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/adaptor/ProtoConverter.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/adaptor/ProtoConverter.java @@ -475,13 +475,17 @@ public static ClientSubscriptionsProto convertToClientSubscriptionsProto(Collect if (topicSubscription instanceof IntegrationTopicSubscription) { source = SubscriptionsSourceProto.INTEGRATION; } - topicSubscriptionsProto.add(topicSubscription.getShareName() == null ? - getTopicSubscriptionProto(topicSubscription) : - getTopicSubscriptionProtoWithShareName(topicSubscription)); + topicSubscriptionsProto.add(toTopicSubscriptionProto(topicSubscription)); } return ClientSubscriptionsProto.newBuilder().addAllSubscriptions(topicSubscriptionsProto).setSource(source).build(); } + public static TopicSubscriptionProto toTopicSubscriptionProto(TopicSubscription topicSubscription) { + return topicSubscription.getShareName() == null ? + getTopicSubscriptionProto(topicSubscription) : + getTopicSubscriptionProtoWithShareName(topicSubscription); + } + private static SubscriptionOptionsProto prepareOptionsProto(TopicSubscription topicSubscription) { if (topicSubscription.getOptions() == null) { return SubscriptionOptionsProto.getDefaultInstance(); diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/server/MqttSessionHandler.java b/application/src/main/java/org/thingsboard/mqtt/broker/server/MqttSessionHandler.java index 627706dcd..fef4f9074 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/server/MqttSessionHandler.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/server/MqttSessionHandler.java @@ -109,10 +109,13 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { address = getAddress(ctx); clientSessionCtx.setAddress(address); } + if (clientSessionCtx.getChannel() == null) { + // The ChannelHandlerContext is stable for the connection; capture it once rather than on every message. + clientSessionCtx.setChannel(ctx); + } if (log.isTraceEnabled()) { log.trace("[{}][{}][{}] Processing msg: {}", address, clientId, sessionId, msg); } - clientSessionCtx.setChannel(ctx); try { if (!(msg instanceof MqttMessage message)) { log.warn("[{}][{}] Received unknown message", clientId, sessionId); diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/entity/integration/DefaultTbIntegrationService.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/entity/integration/DefaultTbIntegrationService.java index 98592cea9..3a779f98f 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/entity/integration/DefaultTbIntegrationService.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/entity/integration/DefaultTbIntegrationService.java @@ -15,16 +15,21 @@ */ package org.thingsboard.mqtt.broker.service.entity.integration; +import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.mqtt.broker.common.data.User; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.dao.integration.IntegrationService; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; +import org.thingsboard.mqtt.broker.gen.queue.InternodeNotificationProto; import org.thingsboard.mqtt.broker.service.entity.AbstractTbEntityService; import org.thingsboard.mqtt.broker.service.integration.PlatformIntegrationService; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; +import org.thingsboard.mqtt.broker.service.notification.InternodeNotificationsService; @Slf4j @Service @@ -34,12 +39,17 @@ public class DefaultTbIntegrationService extends AbstractTbEntityService impleme private final IntegrationService integrationService; private final PlatformIntegrationService platformIntegrationService; private final RateLimitService rateLimitService; + private final InternodeNotificationsService internodeNotificationsService; @Override public Integration save(Integration integration, User currentUser) { boolean created = integration.getId() == null; Integration result = integrationService.saveIntegration(integration); platformIntegrationService.processIntegrationUpdate(result, created); + internodeNotificationsService.broadcast( + InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(toLifecycleConfigProto(result)) + .build()); return result; } @@ -50,6 +60,13 @@ public void delete(Integration integration, User currentUser) { rateLimitService.decrementApplicationClientsCount(); } platformIntegrationService.processIntegrationDelete(integration, removed); + internodeNotificationsService.broadcast( + InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId(integration.getIdStr()) + .setDeleted(true) + .build()) + .build()); } @Override @@ -57,4 +74,16 @@ public void restart(Integration integration, User currentUser) throws Thingsboar platformIntegrationService.processIntegrationRestart(integration); } + private IntegrationLifecycleConfigProto toLifecycleConfigProto(Integration integration) { + IntegrationLifecycleConfigProto.Builder builder = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId(integration.getIdStr()) + .setDeleted(false); + JsonNode configuration = integration.getConfiguration(); + if (configuration != null && configuration.has(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY)) { + configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY) + .forEach(node -> builder.addLifecycleEventTypes(node.asText())); + } + return builder.build(); + } + } diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/AuthorizationAction.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/AuthorizationAction.java new file mode 100644 index 000000000..6c9022389 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/AuthorizationAction.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Operation that was denied during authorization, carried in the {@code action} field of a + * CLIENT_AUTHORIZATION_FAILED lifecycle event. The {@link #label} is the lowercase wire value emitted to + * integrations, keeping callers from hand-typing the literal. + */ +@Getter +@RequiredArgsConstructor +public enum AuthorizationAction { + + PUBLISH("publish"), + SUBSCRIBE("subscribe"); + + private final String label; + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationService.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationService.java index 8ce670581..b6a75abc0 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationService.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationService.java @@ -32,6 +32,7 @@ import org.thingsboard.mqtt.broker.common.data.event.LifecycleEvent; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardErrorCode; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; import org.thingsboard.mqtt.broker.common.data.integration.ComponentLifecycleEvent; import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.common.data.subscription.IntegrationTopicSubscription; @@ -122,15 +123,28 @@ public void processServiceInfo(ServiceInfo serviceInfo) { @Override public void updateSubscriptions(Integration integration) { JsonNode configuration = integration.getConfiguration(); - if (!configuration.has("topicFilters")) { - log.error("[{}][{}] Topic filters not configured", integration.getId(), integration.getName()); + + JsonNode topicFilters = configuration.get("topicFilters"); + JsonNode lifecycleEventTypes = configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY); + + boolean hasTopicFilters = topicFilters != null && topicFilters.isArray() && !topicFilters.isEmpty(); + boolean hasLifecycleEvents = lifecycleEventTypes != null && !lifecycleEventTypes.isEmpty(); + + if (!hasTopicFilters && !hasLifecycleEvents) { + log.error("[{}][{}] Neither topic filters nor lifecycle event types are configured", + integration.getId(), integration.getName()); return; } - ArrayNode topicFiltersArrayNode = (ArrayNode) configuration.get("topicFilters"); - Set subscriptions = Sets.newHashSetWithExpectedSize(topicFiltersArrayNode.size()); - topicFiltersArrayNode.forEach(topicFilter -> subscriptions.add(new IntegrationTopicSubscription(topicFilter.asText()))); - integrationSubscriptionUpdateService.processSubscriptionsUpdate(integration.getIdStr(), subscriptions); + if (hasTopicFilters) { + ArrayNode topicFiltersArrayNode = (ArrayNode) topicFilters; + + Set subscriptions = Sets.newHashSetWithExpectedSize(topicFiltersArrayNode.size()); + topicFiltersArrayNode.forEach(topicFilter -> subscriptions.add(new IntegrationTopicSubscription(topicFilter.asText()))); + integrationSubscriptionUpdateService.processSubscriptionsUpdate(integration.getIdStr(), subscriptions); + } else { + integrationSubscriptionUpdateService.processSubscriptionsUpdate(integration.getIdStr(), Collections.emptySet()); + } } @Override diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisher.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisher.java new file mode 100644 index 000000000..568045bf4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisher.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import org.thingsboard.mqtt.broker.common.data.SessionInfo; +import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; +import org.thingsboard.mqtt.broker.session.ClientSessionCtx; +import org.thingsboard.mqtt.broker.session.DisconnectReasonType; + +import java.util.List; + +public interface IntegrationLifecycleEventPublisher { + + void publishConnected(ClientSessionCtx ctx); + + void publishDisconnected(ClientSessionCtx ctx, DisconnectReasonType reasonType); + + void publishSubscribed(ClientSessionCtx ctx, List subscriptions); + + void publishUnsubscribed(ClientSessionCtx ctx, List subscriptions); + + void publishAuthenticationFailed(ClientSessionCtx ctx, String clientId, String reason); + + void publishAuthorizationDenied(ClientSessionCtx ctx, AuthorizationAction action, String topic); + + void publishConnectionFailed(ClientSessionCtx ctx, SessionInfo sessionInfo, String reason); + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImpl.java new file mode 100644 index 000000000..88f1ae80c --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImpl.java @@ -0,0 +1,262 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import io.netty.handler.codec.mqtt.MqttVersion; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.mqtt.broker.adaptor.ProtoConverter; +import org.thingsboard.mqtt.broker.common.data.SessionInfo; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; +import org.thingsboard.mqtt.broker.common.data.util.BytesUtil; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.gen.queue.TopicSubscriptionProto; +import org.thingsboard.mqtt.broker.queue.cluster.ServiceInfoProvider; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.service.mqtt.persistence.integration.IntegrationEventMsgQueuePublisher; +import org.thingsboard.mqtt.broker.service.processing.PublishMsgCallback; +import org.thingsboard.mqtt.broker.service.stats.DroppedLifecycleEventStats; +import org.thingsboard.mqtt.broker.service.stats.StatsManager; +import org.thingsboard.mqtt.broker.session.ClientSessionCtx; +import org.thingsboard.mqtt.broker.session.DisconnectReasonType; +import org.thingsboard.mqtt.broker.util.MqttReasonCodeResolver; + +import java.util.List; +import java.util.Set; + +@Slf4j +@Service +@RequiredArgsConstructor +public class IntegrationLifecycleEventPublisherImpl implements IntegrationLifecycleEventPublisher { + + private final IntegrationLifecycleEventTypeCache lifecycleEventTypeCache; + private final IntegrationEventMsgQueuePublisher integrationEventMsgQueuePublisher; + private final StatsManager statsManager; + private final ServiceInfoProvider serviceInfoProvider; + + private DroppedLifecycleEventStats droppedLifecycleEventStats; + + // Reused across all sends (no per-event allocation). A real callback ensures asynchronous Kafka send failures + // — the common failure mode — increment the dropped-event metric; PublishMsgCallback.EMPTY would swallow them + // silently. The detailed per-integration error is already logged by IntegrationEventMsgQueuePublisher. + private PublishMsgCallback droppedEventCallback; + + @PostConstruct + public void init() { + this.droppedLifecycleEventStats = statsManager.getDroppedLifecycleEventStats(); + this.droppedEventCallback = new PublishMsgCallback() { + @Override + public void onSuccess() { + } + + @Override + public void onFailure(Throwable t) { + droppedLifecycleEventStats.increment(); + } + }; + } + + @Override + public void publishConnected(ClientSessionCtx ctx) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED); + if (integrationIds.isEmpty()) { + return; + } + int protocolVersion = ctx.getMqttVersion() != null ? ctx.getMqttVersion().protocolLevel() : 0; + ClientLifecycleEventMsgProto proto = newSessionBuilder(ctx, ClientLifecycleEventType.CLIENT_CONNECTED) + .setCleanStart(ctx.getSessionInfo().isCleanStart()) + .setKeepAlive(ctx.getSessionInfo().getKeepAlive()) + .setProtocolVersion(protocolVersion) + .setSessionExpiryInterval(ctx.getSessionInfo().safeGetSessionExpiryInterval()) + .build(); + publish(integrationIds, proto); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_CONNECTED, t); + } + } + + @Override + public void publishDisconnected(ClientSessionCtx ctx, DisconnectReasonType reasonType) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_DISCONNECTED); + if (integrationIds.isEmpty()) { + return; + } + ClientLifecycleEventMsgProto proto = newSessionBuilder(ctx, ClientLifecycleEventType.CLIENT_DISCONNECTED) + .setDisconnectReason(MqttReasonCodeResolver.disconnect(reasonType).name()) + .build(); + publish(integrationIds, proto); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_DISCONNECTED, t); + } + } + + @Override + public void publishSubscribed(ClientSessionCtx ctx, List subscriptions) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED); + if (integrationIds.isEmpty()) { + return; + } + ClientLifecycleEventMsgProto.Builder builder = newSessionBuilder(ctx, ClientLifecycleEventType.CLIENT_SUBSCRIBED); + for (TopicSubscription sub : subscriptions) { + builder.addSubscriptions(ProtoConverter.toTopicSubscriptionProto(sub)); + } + publish(integrationIds, builder.build()); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_SUBSCRIBED, t); + } + } + + @Override + public void publishUnsubscribed(ClientSessionCtx ctx, List subscriptions) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED); + if (integrationIds.isEmpty()) { + return; + } + ClientLifecycleEventMsgProto.Builder builder = newSessionBuilder(ctx, ClientLifecycleEventType.CLIENT_UNSUBSCRIBED); + for (TopicSubscription sub : subscriptions) { + // An UNSUBSCRIBE names filters, not settings; carry only the removed subscription's identity + // (bare topic filter + shareName when shared) so $share// can be reconstructed. + TopicSubscriptionProto.Builder subProto = TopicSubscriptionProto.newBuilder().setTopic(sub.getTopicFilter()); + if (sub.getShareName() != null) { + subProto.setShareName(sub.getShareName()); + } + builder.addSubscriptions(subProto); + } + publish(integrationIds, builder.build()); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED, t); + } + } + + @Override + public void publishAuthenticationFailed(ClientSessionCtx ctx, String clientId, String reason) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_AUTHENTICATION_FAILED); + if (integrationIds.isEmpty()) { + return; + } + String username = ctx.getUsername(); + int protocolVersion = ctx.getMqttVersion() != null ? ctx.getMqttVersion().protocolLevel() : 0; + ClientLifecycleEventMsgProto proto = ClientLifecycleEventMsgProto.newBuilder() + .setEventType(ClientLifecycleEventType.CLIENT_AUTHENTICATION_FAILED.name()) + .setClientId(nullToEmpty(clientId)) + .setUsername(nullToEmpty(username)) + .setSessionId(ctx.getSessionId() != null ? ctx.getSessionId().toString() : "") + .setIpAddress(toIpString(ctx.getAddressBytes())) + .setTs(System.currentTimeMillis()) + .setTbmqNode(serviceInfoProvider.getServiceId()) + .setProtocolVersion(protocolVersion) + .setReason(nullToEmpty(reason)) + .build(); + publish(integrationIds, proto); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_AUTHENTICATION_FAILED, t); + } + } + + @Override + public void publishAuthorizationDenied(ClientSessionCtx ctx, AuthorizationAction action, String topic) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_AUTHORIZATION_FAILED); + if (integrationIds.isEmpty()) { + return; + } + if (ctx.getSessionInfo() == null) { + // Authorization can be checked before the session is established (e.g. last-will topic validation + // at CONNECT, before setSessionInfo). With no session to attribute the event to, skip emission. + return; + } + ClientLifecycleEventMsgProto proto = newSessionBuilder(ctx, ClientLifecycleEventType.CLIENT_AUTHORIZATION_FAILED) + .setAction(action.getLabel()) + .setTopic(topic) + .build(); + publish(integrationIds, proto); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_AUTHORIZATION_FAILED, t); + } + } + + @Override + public void publishConnectionFailed(ClientSessionCtx ctx, SessionInfo sessionInfo, String reason) { + try { + Set integrationIds = lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTION_FAILED); + if (integrationIds.isEmpty()) { + return; + } + // The session is built by the time a connection is refused, but it is not yet set on the ctx for + // pre-connection validation failures (they run before ctx.setSessionInfo), so it is passed explicitly. + ClientLifecycleEventMsgProto proto = newSessionBuilder(ctx, sessionInfo, ClientLifecycleEventType.CLIENT_CONNECTION_FAILED) + .setReason(reason) + .build(); + publish(integrationIds, proto); + } catch (Throwable t) { + onPublishError(ClientLifecycleEventType.CLIENT_CONNECTION_FAILED, t); + } + } + + /** + * Builds a base event proto from the established session. Callers MUST ensure the session is set + * (i.e. {@code ctx.getSessionInfo() != null} and its client info is populated) before invoking this; + * it dereferences both without null checks. Pre-session callers (e.g. last-will validation at CONNECT) + * must guard {@code getSessionInfo() == null} themselves rather than relying on it here. + */ + private ClientLifecycleEventMsgProto.Builder newSessionBuilder(ClientSessionCtx ctx, ClientLifecycleEventType eventType) { + return newSessionBuilder(ctx, ctx.getSessionInfo(), eventType); + } + + private ClientLifecycleEventMsgProto.Builder newSessionBuilder(ClientSessionCtx ctx, SessionInfo sessionInfo, ClientLifecycleEventType eventType) { + return ClientLifecycleEventMsgProto.newBuilder() + .setEventType(eventType.name()) + .setClientId(sessionInfo.getClientInfo().getClientId()) + .setUsername(nullToEmpty(ctx.getUsername())) + .setClientCertCn(nullToEmpty(ctx.getClientCertCn())) + .setSessionId(sessionInfo.getSessionId().toString()) + .setIpAddress(toIpString(sessionInfo.getClientInfo().getClientIpAdr())) + .setTs(System.currentTimeMillis()) + .setTbmqNode(sessionInfo.getServiceId()); + } + + private static String nullToEmpty(String s) { + return s == null ? "" : s; + } + + private void publish(Set integrationIds, ClientLifecycleEventMsgProto lifecycleMsg) { + for (String integrationId : integrationIds) { + // No record key: the event stream uses a single partition (0) with delete-based retention, so the key + // plays no role in partition routing, compaction, or consumer-side dedup (the consumer assigns its own + // packet id). Skipping it avoids a per-event UUID allocation. Send failures are counted via the callback. + integrationEventMsgQueuePublisher.sendEventMsg(integrationId, new TbProtoQueueMsg<>(lifecycleMsg), droppedEventCallback); + } + } + + private void onPublishError(ClientLifecycleEventType eventType, Throwable t) { + log.warn("Failed to publish lifecycle event [{}]; dropping.", eventType, t); + droppedLifecycleEventStats.increment(); + } + + private String toIpString(byte[] ipAdr) { + // Single-source IPv6/error handling via BytesUtil; keep null/empty as "" for address-less sessions. + return (ipAdr == null || ipAdr.length == 0) ? "" : BytesUtil.toHostAddress(ipAdr); + } + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCache.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCache.java new file mode 100644 index 000000000..c89b43c16 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCache.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; + +import java.util.Set; + +public interface IntegrationLifecycleEventTypeCache { + + void put(String integrationId, Set eventTypes); + + void remove(String integrationId); + + Set getIntegrationIds(ClientLifecycleEventType eventType); + + /** + * Applies an integration lifecycle-config notification (delete -> {@link #remove}, otherwise parse the event-type + * names and {@link #put}). Owns the delete/parse semantics so the broadcast-to-self and consume paths share one + * implementation instead of duplicating it. + */ + void processIntegrationLifecycleConfig(IntegrationLifecycleConfigProto proto); + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImpl.java new file mode 100644 index 000000000..36ffe648b --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImpl.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; + +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Slf4j +@Service +public class IntegrationLifecycleEventTypeCacheImpl implements IntegrationLifecycleEventTypeCache { + + // Mutations are serialized by the instance monitor (put/remove/rebuildReverseIndex all run under synchronized); + // reads never touch this map - they use the immutable byEventType snapshot below. A plain HashMap suffices: the + // synchronization, not the map type, provides thread-safety. + private final Map> byIntegrationId = new HashMap<>(); + + private volatile Map> byEventType = Map.of(); + + @Override + public synchronized void put(String integrationId, Set eventTypes) { + if (eventTypes == null || eventTypes.isEmpty()) { + byIntegrationId.remove(integrationId); + } else { + byIntegrationId.put(integrationId, Set.copyOf(eventTypes)); + } + rebuildReverseIndex(); + } + + @Override + public synchronized void remove(String integrationId) { + byIntegrationId.remove(integrationId); + rebuildReverseIndex(); + } + + @Override + public Set getIntegrationIds(ClientLifecycleEventType eventType) { + return byEventType.getOrDefault(eventType, Set.of()); + } + + @Override + public void processIntegrationLifecycleConfig(IntegrationLifecycleConfigProto proto) { + if (proto.getDeleted()) { + remove(proto.getIntegrationId()); + return; + } + Set eventTypes = ClientLifecycleEventTypeUtil.parse( + proto.getLifecycleEventTypesList(), + name -> log.warn("[{}] Unknown lifecycle event type: {}", proto.getIntegrationId(), name)); + put(proto.getIntegrationId(), eventTypes); + } + + private void rebuildReverseIndex() { + Map> mutable = new HashMap<>(); + byIntegrationId.forEach((integrationId, eventTypes) -> + eventTypes.forEach(eventType -> + mutable.computeIfAbsent(eventType, k -> new HashSet<>()).add(integrationId))); + Map> immutable = new EnumMap<>(ClientLifecycleEventType.class); + mutable.forEach((eventType, ids) -> immutable.put(eventType, Set.copyOf(ids))); + this.byEventType = immutable; + } + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisher.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisher.java new file mode 100644 index 000000000..1187e7077 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisher.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.mqtt.persistence.integration; + +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.service.processing.PublishMsgCallback; + +public interface IntegrationEventMsgQueuePublisher { + + void sendEventMsg(String integrationId, TbProtoQueueMsg queueMsg, PublishMsgCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImpl.java new file mode 100644 index 000000000..659112981 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImpl.java @@ -0,0 +1,85 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.mqtt.persistence.integration; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.queue.TbQueueCallback; +import org.thingsboard.mqtt.broker.queue.TbQueueMsgMetadata; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.queue.provider.integration.IntegrationMsgQueueProvider; +import org.thingsboard.mqtt.broker.queue.publish.TbPublishServiceImpl; +import org.thingsboard.mqtt.broker.service.analysis.ClientLogger; +import org.thingsboard.mqtt.broker.service.processing.PublishMsgCallback; +import org.thingsboard.mqtt.broker.service.util.IntegrationHelperService; + +@Slf4j +@Service +@RequiredArgsConstructor +public class IntegrationEventMsgQueuePublisherImpl implements IntegrationEventMsgQueuePublisher { + + private final ClientLogger clientLogger; + private final IntegrationMsgQueueProvider msgQueueProvider; + private final IntegrationHelperService integrationHelperService; + + private final boolean isTraceEnabled = log.isTraceEnabled(); + + private TbPublishServiceImpl publisher; + + @PostConstruct + public void init() { + this.publisher = TbPublishServiceImpl.builder() + .queueName("ieEventMsg") + .producer(msgQueueProvider.getIeEventMsgProducer()) + .partition(0) + .build(); + this.publisher.init(); + } + + @PreDestroy + public void destroy() { + this.publisher.destroy(); + } + + @Override + public void sendEventMsg(String integrationId, TbProtoQueueMsg queueMsg, PublishMsgCallback callback) { + clientLogger.logEvent(integrationId, this.getClass(), "Start waiting for IE event msg to be persisted"); + String ieEventQueueTopic = integrationHelperService.getIntegrationEventTopic(integrationId); + this.publisher.send(queueMsg, + new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + clientLogger.logEvent(integrationId, IntegrationEventMsgQueuePublisherImpl.this.getClass(), "Persisted event msg in IE event Queue"); + if (isTraceEnabled) { + log.trace("[{}] Successfully sent lifecycle event msg to the ie event queue.", integrationId); + } + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to send lifecycle event msg to the ie event queue.", integrationId, t); + callback.onFailure(t); + } + }, + ieEventQueueTopic); + } + +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationMsgQueuePublisherImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationMsgQueuePublisherImpl.java index 0505b1c6f..71a0a2d2b 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationMsgQueuePublisherImpl.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationMsgQueuePublisherImpl.java @@ -66,7 +66,7 @@ public void sendMsg(String integrationId, TbProtoQueueMsg> internodeNotificationsProducer; @@ -74,6 +76,11 @@ public void broadcast(InternodeNotificationProto notificationProto) { if (notificationProto.hasClientSessionStatsCleanupProto()) { log.trace("[{}] Forwarding message to local MQTT client session stats cleanup processor {}", serviceId, notificationProto.getClientSessionStatsCleanupProto()); clientSessionStatsCleanupProcessor.processClientSessionStatsCleanup(notificationProto.getClientSessionStatsCleanupProto()); + continue; + } + if (notificationProto.hasIntegrationLifecycleConfigProto()) { + log.trace("[{}] Forwarding message to local integration lifecycle event type cache {}", serviceId, notificationProto.getIntegrationLifecycleConfigProto()); + integrationLifecycleEventTypeCache.processIntegrationLifecycleConfig(notificationProto.getIntegrationLifecycleConfigProto()); } } } diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/processing/PublishMsgCallback.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/processing/PublishMsgCallback.java index 7f4700526..027bb2818 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/processing/PublishMsgCallback.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/processing/PublishMsgCallback.java @@ -17,6 +17,14 @@ public interface PublishMsgCallback { + PublishMsgCallback EMPTY = new PublishMsgCallback() { + @Override + public void onSuccess() {} + + @Override + public void onFailure(Throwable t) {} + }; + void onSuccess(); default void onBatchSuccess(int totalCount) { diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DefaultDroppedLifecycleEventStats.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DefaultDroppedLifecycleEventStats.java new file mode 100644 index 000000000..e7927f29e --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DefaultDroppedLifecycleEventStats.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.stats; + +import org.thingsboard.mqtt.broker.common.stats.DefaultCounter; +import org.thingsboard.mqtt.broker.common.stats.StatsConstantNames; +import org.thingsboard.mqtt.broker.common.stats.StatsFactory; + +public class DefaultDroppedLifecycleEventStats implements DroppedLifecycleEventStats { + + private final DefaultCounter droppedLifecycleEventsCounter; + + public DefaultDroppedLifecycleEventStats(StatsFactory statsFactory) { + this.droppedLifecycleEventsCounter = statsFactory.createDefaultCounter(StatsConstantNames.DROPPED_LIFECYCLE_EVENTS); + } + + @Override + public void increment() { + droppedLifecycleEventsCounter.increment(); + } + + @Override + public void increment(int count) { + droppedLifecycleEventsCounter.add(count); + } + + @Override + public int getCount() { + return droppedLifecycleEventsCounter.get(); + } + + @Override + public void reset() { + droppedLifecycleEventsCounter.clear(); + } +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DroppedLifecycleEventStats.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DroppedLifecycleEventStats.java new file mode 100644 index 000000000..c5ae9f21c --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/DroppedLifecycleEventStats.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.stats; + +/** + * Tracks dropped lifecycle events as a cumulative {@code droppedLifecycleEvents} Prometheus counter. + * Obtained from {@link StatsManager}, so it shares the {@code stats.enabled} master switch with every other + * broker metric; when stats are disabled the stub implementation is a no-op and nothing is exposed. + */ +public interface DroppedLifecycleEventStats { + + void increment(); + + void increment(int count); + + /** + * Number of lifecycle events dropped since the last {@link #reset()} — read for the periodic stats log line. + */ + int getCount(); + + /** + * Clears the per-interval count read by {@link #getCount()}. The cumulative {@code droppedLifecycleEvents} + * Prometheus counter is unaffected and stays monotonic. + */ + void reset(); +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManager.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManager.java index fb6ca42cc..8312b3bc6 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManager.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManager.java @@ -42,6 +42,12 @@ public interface StatsManager { */ DroppedMsgStats getDroppedMsgStats(); + /** + * Returns the dropped-lifecycle-events stats. Shares the {@code stats.enabled} master switch; when stats + * are disabled the stub manager returns a no-op instance. + */ + DroppedLifecycleEventStats getDroppedLifecycleEventStats(); + ClientSessionEventConsumerStats createClientSessionEventConsumerStats(String consumerId); PublishMsgConsumerStats createPublishMsgConsumerStats(String consumerId); diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerImpl.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerImpl.java index cf0b26c53..be5f76d48 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerImpl.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerImpl.java @@ -83,6 +83,7 @@ public class StatsManagerImpl implements StatsManager, ActorStatsManager, SqlQue private ClientActorStats clientActorStats; private FlowControlStats flowControlStats; private DroppedMsgStats droppedMsgStats; + private DroppedLifecycleEventStats droppedLifecycleEventStats; @Value("${stats.application-processor.enabled}") private boolean applicationProcessorStatsEnabled; @@ -100,6 +101,7 @@ public void init() { gauges.add(new Gauge(StatsType.FLOW_CONTROL.getPrintName() + ".inflightCount", defaultFlowControlStats::getInflightCount)); gauges.add(new Gauge(StatsType.FLOW_CONTROL.getPrintName() + ".delayedQueueSize", defaultFlowControlStats::getDelayedQueueSize)); this.droppedMsgStats = new DefaultDroppedMsgStats(statsFactory); + this.droppedLifecycleEventStats = new DefaultDroppedLifecycleEventStats(statsFactory); } @PreDestroy @@ -126,6 +128,11 @@ public DroppedMsgStats getDroppedMsgStats() { return droppedMsgStats; } + @Override + public DroppedLifecycleEventStats getDroppedLifecycleEventStats() { + return droppedLifecycleEventStats; + } + @Override public ClientSessionEventConsumerStats createClientSessionEventConsumerStats(String consumerId) { log.trace("Creating ClientSessionEventConsumerStats, consumerId - {}", consumerId); @@ -469,6 +476,9 @@ public void printStats() { log.info("[{}] Stats: count = [{}]", BrokerConstants.DROPPED_MSGS, droppedMsgStats.getCount()); droppedMsgStats.reset(); + log.info("[{}] Stats: count = [{}]", StatsConstantNames.DROPPED_LIFECYCLE_EVENTS, droppedLifecycleEventStats.getCount()); + droppedLifecycleEventStats.reset(); + StringBuilder gaugeLogBuilder = new StringBuilder(); for (Gauge gauge : gauges) { gaugeLogBuilder.append(gauge.getName()).append(" = [").append(gauge.getValueSupplier().get().intValue()).append("] "); diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerStub.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerStub.java index 509f0db8a..e2918cee0 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerStub.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StatsManagerStub.java @@ -60,6 +60,11 @@ public DroppedMsgStats getDroppedMsgStats() { return StubDroppedMsgStats.STUB_DROPPED_MSG_STATS; } + @Override + public DroppedLifecycleEventStats getDroppedLifecycleEventStats() { + return StubDroppedLifecycleEventStats.STUB_DROPPED_LIFECYCLE_EVENT_STATS; + } + @Override public ClientSessionEventConsumerStats createClientSessionEventConsumerStats(String consumerId) { return StubClientSessionEventConsumerStats.STUB_CLIENT_SESSION_EVENT_CONSUMER_STATS; diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StubDroppedLifecycleEventStats.java b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StubDroppedLifecycleEventStats.java new file mode 100644 index 000000000..261f37e55 --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/service/stats/StubDroppedLifecycleEventStats.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.stats; + +public class StubDroppedLifecycleEventStats implements DroppedLifecycleEventStats { + + public static final StubDroppedLifecycleEventStats STUB_DROPPED_LIFECYCLE_EVENT_STATS = new StubDroppedLifecycleEventStats(); + + @Override + public void increment() { + } + + @Override + public void increment(int count) { + } + + @Override + public int getCount() { + return 0; + } + + @Override + public void reset() { + } +} diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/session/ClientSessionCtx.java b/application/src/main/java/org/thingsboard/mqtt/broker/session/ClientSessionCtx.java index f444b220d..dbfb65433 100644 --- a/application/src/main/java/org/thingsboard/mqtt/broker/session/ClientSessionCtx.java +++ b/application/src/main/java/org/thingsboard/mqtt/broker/session/ClientSessionCtx.java @@ -20,7 +20,10 @@ import io.netty.handler.codec.mqtt.MqttPublishMessage; import io.netty.handler.codec.mqtt.MqttVersion; import io.netty.handler.ssl.SslHandler; -import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.mqtt.broker.actors.client.state.PubResponseProcessingCtx; import org.thingsboard.mqtt.broker.actors.client.state.PublishedInFlightCtx; @@ -40,10 +43,40 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +/** + * Mutable per-connection state for a single MQTT client. One instance exists per client connection — over any + * transport: plain TCP, TLS, WebSocket, or WebSocket Secure (see {@link #initializerName}) — created in + * {@code MqttSessionHandler}; once the client is registered it is held as a value in + * {@code ClientSessionCtxService} keyed by clientId. + * + *

Initialization phases. Most fields are populated after construction, in stages and on two + * threads, so accessing a not-yet-populated field can return {@code null} or throw. The main fields, grouped + * by the thread that writes them: + *

    + *
  • Netty I/O thread — at construction: {@link #sessionId}, {@link #sslHandler}, + * {@link #initializerName}; on the first inbound bytes: {@link #address} and {@link #channel}; while the + * CONNECT packet is parsed: {@link #mqttVersion} and, for MQTT 5 enhanced authentication, the + * {@link #enhancedAuthState} auth method and buffered CONNECT.
    • + *
  • Client actor thread — while the CONNECT/AUTH exchange is processed: {@link #sessionInfo}, + * {@link #authRulePatterns}, {@link #clientType}, the authentication results ({@link #username}, + * {@link #authDetails}, {@link #clientCertCn}), the {@link #enhancedAuthState} SCRAM server, and + * {@link #publishedInFlightCtx}.
    • + *
+ * Accessors that dereference such a field (e.g. {@link #getClientId()}, {@link #isCleanSession()}, + * {@link #getAddressBytes()}, {@link #isWritable()}) are only safe once it has been set. + * + *

Concurrency. Because these fields are written and read across the Netty I/O, client actor and + * message-delivery threads, they are {@code volatile} (visibility only). Compound updates that need + * atomicity use dedicated types (see {@link #nonWritableCounted}). + */ @Slf4j -@Data +@Getter +@Setter +@ToString(of = "sessionId") +@EqualsAndHashCode(of = "sessionId") public class ClientSessionCtx implements SessionContext { + // Final: assigned once at construction and never reassigned. private final UUID sessionId; private final SslHandler sslHandler; private final String initializerName; @@ -53,22 +86,22 @@ public class ClientSessionCtx implements SessionContext { // Tracks whether this session is currently counted in the broker-wide nonWritableClientsCount gauge. // Used to make increment/decrement idempotent and to allow decrement on abrupt disconnect. private final AtomicBoolean nonWritableCounted = new AtomicBoolean(false); + // Enhanced-auth (MQTT 5 SCRAM) handshake working set; partially cleared once it completes. + private final EnhancedAuthState enhancedAuthState = new EnhancedAuthState(); + // Assigned after construction (see the "Initialization phases" above); volatile for cross-thread visibility. + private volatile InetSocketAddress address; + private volatile ChannelHandlerContext channel; private volatile SessionInfo sessionInfo; private volatile List authRulePatterns; private volatile ClientType clientType; private volatile MqttVersion mqttVersion; - private volatile InetSocketAddress address; private volatile TopicAliasCtx topicAliasCtx; private volatile PublishedInFlightCtx publishedInFlightCtx; - private volatile MqttConnectMessage connectMsgFromEnhancedAuth; - private volatile String authMethod; - private volatile ScramServerWithCallbackHandler scramServerWithCallbackHandler; + private volatile String username; private volatile String authDetails; private volatile String clientCertCn; - private ChannelHandlerContext channel; - public ClientSessionCtx() { this(null, UUID.randomUUID(), null, BrokerConstants.TCP); } @@ -96,6 +129,44 @@ public String getClientId() { return (sessionInfo != null && sessionInfo.getClientInfo() != null) ? sessionInfo.getClientId() : null; } + // --- Enhanced-auth handshake: thin delegation; the state lives in EnhancedAuthState --- + + public String getAuthMethod() { + return enhancedAuthState.getAuthMethod(); + } + + public void setAuthMethod(String authMethod) { + enhancedAuthState.setAuthMethod(authMethod); + } + + public ScramServerWithCallbackHandler getScramServerWithCallbackHandler() { + return enhancedAuthState.getScramServer(); + } + + public void setScramServerWithCallbackHandler(ScramServerWithCallbackHandler scramServer) { + enhancedAuthState.setScramServer(scramServer); + } + + public MqttConnectMessage getConnectMsgFromEnhancedAuth() { + return enhancedAuthState.getConnectMsg(); + } + + public void setConnectMsgFromEnhancedAuth(MqttConnectMessage connectMsg) { + enhancedAuthState.setConnectMsg(connectMsg); + } + + public boolean isDefaultAuth() { + return enhancedAuthState.isDefaultAuth(); + } + + public void clearScramServer() { + enhancedAuthState.clearScramServer(); + } + + public void clearConnectMsg() { + enhancedAuthState.clearConnectMsg(); + } + public void initPublishedInFlightCtx(FlowControlService flowControlService, MqttPublishMsgDeliveryService deliveryService, FlowControlStats stats, @@ -124,6 +195,7 @@ public void onChannelWritable() { } } + // publishedInFlightCtx lifecycle (init/release) is driven solely by the client actor thread. public void releasePublishedInFlightCtx() { if (publishedInFlightCtx != null) { publishedInFlightCtx.release(); @@ -148,16 +220,4 @@ public void closeChannel() { channel.flush(); channel.close(); } - - public boolean isDefaultAuth() { - return connectMsgFromEnhancedAuth == null; - } - - public void clearScramServer() { - scramServerWithCallbackHandler = null; - } - - public void clearConnectMsg() { - connectMsgFromEnhancedAuth = null; - } } diff --git a/application/src/main/java/org/thingsboard/mqtt/broker/session/EnhancedAuthState.java b/application/src/main/java/org/thingsboard/mqtt/broker/session/EnhancedAuthState.java new file mode 100644 index 000000000..2ea05c32c --- /dev/null +++ b/application/src/main/java/org/thingsboard/mqtt/broker/session/EnhancedAuthState.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.session; + +import io.netty.handler.codec.mqtt.MqttConnectMessage; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.mqtt.broker.service.auth.enhanced.ScramServerWithCallbackHandler; + +/** + * Working set for the MQTT 5 enhanced-authentication (SCRAM) handshake of a single session. + * + *

Lifecycle: {@code authMethod} is set when the initial CONNECT requests enhanced auth and is retained + * for the lifetime of the session (re-AUTH must use the same method). {@code scramServer} and + * {@code connectMsg} are the transient handshake state and are dropped via {@link #clearScramServer()} / + * {@link #clearConnectMsg()} once the handshake completes. {@link #isDefaultAuth()} reports whether the + * session is using plain (non-enhanced) auth, i.e. no enhanced-auth CONNECT was buffered. + * + *

Fields are {@code volatile}: they are written on the Netty I/O thread (initial CONNECT) and the client + * actor thread (AUTH continuation), and read from both. + */ +@Getter +@Setter +public class EnhancedAuthState { + + private volatile String authMethod; + private volatile ScramServerWithCallbackHandler scramServer; + private volatile MqttConnectMessage connectMsg; + + public boolean isDefaultAuth() { + return connectMsg == null; + } + + public void clearScramServer() { + scramServer = null; + } + + public void clearConnectMsg() { + connectMsg = null; + } +} diff --git a/application/src/main/resources/thingsboard-mqtt-broker.yml b/application/src/main/resources/thingsboard-mqtt-broker.yml index d1220cf2a..1b6c5ed01 100644 --- a/application/src/main/resources/thingsboard-mqtt-broker.yml +++ b/application/src/main/resources/thingsboard-mqtt-broker.yml @@ -620,6 +620,17 @@ queue: additional-consumer-config: "${TB_KAFKA_IE_MSG_ADDITIONAL_CONSUMER_CONFIG:max.poll.records:50}" # Additional Kafka producer configs separated by semicolon for the `tbmq.msg.ie` topics. additional-producer-config: "${TB_KAFKA_IE_MSG_ADDITIONAL_PRODUCER_CONFIG:}" + integration-event: + # Kafka topic properties separated by semicolon for the dedicated `tbmq.ie.event` lifecycle-event topics. + # Retention defaults shorter than the data topic since events are best-effort hints. + topic-properties: "${TB_KAFKA_IE_EVENT_MSG_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:26214400;retention.bytes:104857600;partitions:1;replication.factor:1}" + # Additional Kafka consumer configs separated by semicolon for the `tbmq.ie.event` topics. + additional-consumer-config: "${TB_KAFKA_IE_EVENT_MSG_ADDITIONAL_CONSUMER_CONFIG:max.poll.records:50}" + # Additional Kafka producer configs separated by semicolon for the `tbmq.ie.event` topics. Lifecycle events are + # sent synchronously on the MQTT processing thread (same path as a regular publish), so a slow or unavailable Kafka + # can stall that thread for up to max.block.ms before the send fails, after which the event is dropped and counted. + # Lower max.block.ms to tighten that bound at the cost of more dropped events under Kafka pressure. + additional-producer-config: "${TB_KAFKA_IE_EVENT_MSG_ADDITIONAL_PRODUCER_CONFIG:max.block.ms:5000}" internode-notifications: # Prefix for topics used to send system notifications to Broker nodes. topic-prefix: "${TB_KAFKA_INTERNODE_NOTIFICATIONS_TOPIC_PREFIX:tbmq.sys.internode.notifications}" diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImplTest.java index 778c0c3f2..eaeaaa37f 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/ActorProcessorImplTest.java @@ -43,6 +43,7 @@ import org.thingsboard.mqtt.broker.service.auth.AuthorizationRoutingService; import org.thingsboard.mqtt.broker.service.auth.EnhancedAuthenticationService; import org.thingsboard.mqtt.broker.service.auth.enhanced.EnhancedAuthContext; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.auth.enhanced.EnhancedAuthContinueResponse; import org.thingsboard.mqtt.broker.service.auth.enhanced.EnhancedAuthFinalResponse; import org.thingsboard.mqtt.broker.service.auth.providers.AuthResponse; @@ -100,6 +101,7 @@ public class ActorProcessorImplTest { UnauthorizedClientManager unauthorizedClientManager; BlockedClientService blockedClientService; AuthorizationRoutingService authorizationRoutingService; + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; ClientActorState clientActorState; @@ -112,8 +114,9 @@ public void setUp() { unauthorizedClientManager = mock(UnauthorizedClientManager.class); authorizationRoutingService = mock(AuthorizationRoutingService.class); blockedClientService = mock(BlockedClientService.class); + integrationLifecycleEventPublisher = mock(IntegrationLifecycleEventPublisher.class); actorProcessor = spy(new ActorProcessorImpl(disconnectService, enhancedAuthenticationService, - mqttMessageGenerator, clientMqttActorManager, unauthorizedClientManager, blockedClientService, authorizationRoutingService)); + mqttMessageGenerator, clientMqttActorManager, unauthorizedClientManager, blockedClientService, authorizationRoutingService, integrationLifecycleEventPublisher)); clientActorState = new DefaultClientActorState(CLIENT_ID, false, 0); } @@ -405,6 +408,7 @@ public void givenEnhancedAuthStartedSession_whenOnEnhancedAuthContinueAndAuthSuc actorProcessor.onEnhancedAuthContinue(clientActorState, authMsg); + verify(sessionCtxMock).setUsername(any()); verify(sessionCtxMock).setAuthRulePatterns(authorizationRules); verify(sessionCtxMock).setClientType(ClientType.DEVICE); verify(sessionCtxMock).getSessionId(); @@ -442,6 +446,7 @@ public void givenEnhancedAuthStartedSession_whenOnEnhancedAuthContinueAndAuthFai actorProcessor.onEnhancedAuthContinue(clientActorState, authMsg); + verify(sessionCtxMock).setUsername(any()); verify(sessionCtxMock).getChannel(); verify(sessionCtxMock).closeChannel(); verifyNoMoreInteractions(sessionCtxMock); @@ -472,6 +477,7 @@ public void givenEnhancedAuthStartedSession_whenOnEnhancedAuthContinueAndAuthFai actorProcessor.onEnhancedAuthContinue(clientActorState, authMsg); + verify(sessionCtxMock).setUsername(any()); verify(sessionCtxMock).getChannel(); verify(sessionCtxMock).closeChannel(); verifyNoMoreInteractions(sessionCtxMock); diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializerTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializerTest.java index 1f846e37e..c1edb3d08 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializerTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/BrokerInitializerTest.java @@ -34,6 +34,7 @@ import org.thingsboard.mqtt.broker.dao.integration.IntegrationService; import org.thingsboard.mqtt.broker.exception.QueuePersistenceException; import org.thingsboard.mqtt.broker.queue.cluster.ServiceInfoProvider; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventTypeCache; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.client.blocked.BlockedClientService; import org.thingsboard.mqtt.broker.service.mqtt.client.blocked.consumer.BlockedClientConsumerService; @@ -98,6 +99,8 @@ public class BrokerInitializerTest { @MockitoBean IntegrationService integrationService; @MockitoBean + IntegrationLifecycleEventTypeCache lifecycleEventTypeCache; + @MockitoBean ClientsLimitProperties clientsLimitProperties; @MockitoBean ClientSessionEventConsumer clientSessionEventConsumer; diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImplTest.java index 61e84c2ec..315dd533f 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/connect/ConnectServiceImplTest.java @@ -37,6 +37,7 @@ import org.thingsboard.mqtt.broker.common.data.SessionInfo; import org.thingsboard.mqtt.broker.exception.DataValidationException; import org.thingsboard.mqtt.broker.queue.cluster.ServiceInfoProvider; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.PublishMsg; import org.thingsboard.mqtt.broker.service.mqtt.client.event.ClientSessionEventService; @@ -108,6 +109,8 @@ public class ConnectServiceImplTest { MqttPublishMsgDeliveryService mqttPublishMsgDeliveryService; @MockitoBean StatsManager statsManager; + @MockitoBean + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; @MockitoSpyBean ConnectServiceImpl connectService; @@ -162,6 +165,7 @@ public void givenClientSessionContext_whenRefuseConnection_thenVerifyExecutions( verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(eq(CONNECTION_REFUSED_SERVER_UNAVAILABLE)); verify(channelHandlerContext, times(1)).writeAndFlush(any()); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_SERVER_UNAVAILABLE"); } @Test @@ -194,6 +198,7 @@ public void givenPersistentClientWithoutClientId_whenCheckIfProceedConnection_th verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_IDENTIFIER_REJECTED); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_IDENTIFIER_REJECTED"); } @Test @@ -208,6 +213,7 @@ public void givenLastWillMsgInvalid_whenCheckIfProceedConnection_thenConnectionR verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_SERVER_UNAVAILABLE); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_SERVER_UNAVAILABLE"); } @Test @@ -222,6 +228,7 @@ public void givenLastWillMsgNotAuth_whenCheckIfProceedConnection_thenConnectionR verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_NOT_AUTHORIZED"); } @Test @@ -234,6 +241,7 @@ public void givenPersistentClientWithoutClientIdMqtt5_whenCheckIfProceedConnecti verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_CLIENT_IDENTIFIER_NOT_VALID); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_CLIENT_IDENTIFIER_NOT_VALID"); } @Test @@ -249,6 +257,7 @@ public void givenLastWillMsgInvalidMqtt5_whenCheckIfProceedConnection_thenConnec verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_TOPIC_NAME_INVALID); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_TOPIC_NAME_INVALID"); } @Test @@ -264,6 +273,7 @@ public void givenLastWillMsgNotAuthMqtt5_whenCheckIfProceedConnection_thenConnec verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED_5); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_NOT_AUTHORIZED_5"); } @Test @@ -279,6 +289,7 @@ public void givenMqtt5ReceiveMaxZero_whenCheckIfProceedConnection_thenConnection verify(mqttMessageGenerator, times(1)).createMqttConnAckMsg(CONNECTION_REFUSED_PROTOCOL_ERROR); verify(clientMqttActorManager, times(1)).disconnect(any(), any()); + verify(integrationLifecycleEventPublisher, times(1)).publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_PROTOCOL_ERROR"); } @Test diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImplTest.java index c0ebca691..0a96da2e3 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/disconnect/DisconnectServiceImplTest.java @@ -35,6 +35,7 @@ import org.thingsboard.mqtt.broker.common.data.SessionInfo; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRuleService; import org.thingsboard.mqtt.broker.service.historical.stats.TbMessageStatsReportClient; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.client.event.ClientSessionEventService; @@ -88,6 +89,8 @@ public class DisconnectServiceImplTest { TbMessageStatsReportClient tbMessageStatsReportClient; @MockitoBean ChannelBackpressureManager channelBackpressureManager; + @MockitoBean + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; @MockitoSpyBean DisconnectServiceImpl disconnectService; @@ -143,6 +146,30 @@ public void givenSessionInfoIsNotNull_whenDisconnectClient_thenDisconnectCleanly verify(mqttMessageGenerator, never()).createDisconnectMsg(any()); } + @Test + public void givenClusterConflictingSession_whenDisconnect_thenEmitLifecycleDisconnectButSkipSessionEvent() { + MqttDisconnectMsg disconnectMsg = newDisconnectMsg(new DisconnectReason(DisconnectReasonType.ON_CLUSTER_CONFLICTING_SESSIONS)); + disconnectService.disconnect(clientActorState, disconnectMsg); + + // Takeover on another node: the local session-event-service notification must stay suppressed + // (the session is now owned by the new node)... + verify(disconnectService, never()).notifyClientDisconnected(clientActorState, -1, DisconnectReasonType.ON_CLUSTER_CONFLICTING_SESSIONS); + verify(clientSessionEventService, never()).notifyClientDisconnected(any(), any(), any()); + // ...but the lifecycle CLIENT_DISCONNECTED event must still be emitted, to pair with the CLIENT_CONNECTED + // that this node emitted for the now-superseded session. + verify(integrationLifecycleEventPublisher, times(1)).publishDisconnected(ctx, DisconnectReasonType.ON_CLUSTER_CONFLICTING_SESSIONS); + } + + @Test + public void givenConnectionFailure_whenDisconnect_thenDoesNotEmitLifecycleDisconnect() { + MqttDisconnectMsg disconnectMsg = newDisconnectMsg(new DisconnectReason(DisconnectReasonType.ON_CONNECTION_FAILURE)); + disconnectService.disconnect(clientActorState, disconnectMsg); + + // A broker-refused connection never established a session, so the teardown disconnect must NOT emit + // CLIENT_DISCONNECTED (there is no CLIENT_CONNECTED to pair with); CLIENT_CONNECTION_FAILED covers it instead. + verify(integrationLifecycleEventPublisher, never()).publishDisconnected(ctx, DisconnectReasonType.ON_CONNECTION_FAILURE); + } + @Test public void givenMqtt5Client_whenDisconnectClientByServer_thenSendDisconnectMsgAndCloseChannel() { ChannelHandlerContext handlerContext = mock(ChannelHandlerContext.class); diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandlerTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandlerTest.java index 88e1c549b..213dc1c54 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandlerTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttSubscribeHandlerTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.mqtt.broker.actors.client.service.handlers; +import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.mqtt.MqttProperties; import io.netty.handler.codec.mqtt.MqttReasonCodes; import io.netty.handler.codec.mqtt.MqttVersion; @@ -22,6 +23,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; @@ -29,6 +31,7 @@ import org.thingsboard.mqtt.broker.actors.client.messages.mqtt.MqttSubscribeMsg; import org.thingsboard.mqtt.broker.actors.client.service.subscription.ClientSubscriptionService; import org.thingsboard.mqtt.broker.common.data.ApplicationSharedSubscription; +import org.thingsboard.mqtt.broker.common.data.BasicCallback; import org.thingsboard.mqtt.broker.common.data.BrokerConstants; import org.thingsboard.mqtt.broker.common.data.ClientInfo; import org.thingsboard.mqtt.broker.common.data.ClientType; @@ -40,6 +43,8 @@ import org.thingsboard.mqtt.broker.dao.topic.TopicValidationService; import org.thingsboard.mqtt.broker.exception.DataValidationException; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRuleService; +import org.thingsboard.mqtt.broker.service.integration.AuthorizationAction; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.limits.RateLimitService; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.MqttMsgDeliveryService; @@ -69,6 +74,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -99,6 +105,8 @@ public class MqttSubscribeHandlerTest { ApplicationPersistenceProcessor applicationPersistenceProcessor; @MockitoBean RateLimitService rateLimitService; + @MockitoBean + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; @MockitoSpyBean MqttSubscribeHandler mqttSubscribeHandler; @@ -402,6 +410,50 @@ public void givenMqttSubscribeMsg_whenProcessSubscriptions_thenReturnExpectedRes verify(clientSubscriptionService, times(1)).getClientSharedSubscriptions(any()); } + @Test + public void givenDeniedTopics_whenCollectMqttReasonCodes_thenPublishAuthorizationDeniedPerDeniedTopic() { + when(ctx.getMqttVersion()).thenReturn(MqttVersion.MQTT_5); + when(authorizationRuleService.isSubAuthorized(eq("topic1"), any())).thenReturn(false); + when(authorizationRuleService.isSubAuthorized(eq("topic2"), any())).thenReturn(false); + when(authorizationRuleService.isSubAuthorized(eq("topic3"), any())).thenReturn(true); + + MqttSubscribeMsg msg = new MqttSubscribeMsg(UUID.randomUUID(), 1, getTopicSubscriptions()); + mqttSubscribeHandler.collectMqttReasonCodes(ctx, msg); + + // a CLIENT_AUTHORIZATION_FAILED event is emitted once per denied topic, and never for an authorized one + verify(integrationLifecycleEventPublisher, times(1)).publishAuthorizationDenied(ctx, AuthorizationAction.SUBSCRIBE, "topic1"); + verify(integrationLifecycleEventPublisher, times(1)).publishAuthorizationDenied(ctx, AuthorizationAction.SUBSCRIBE, "topic2"); + verify(integrationLifecycleEventPublisher, never()).publishAuthorizationDenied(ctx, AuthorizationAction.SUBSCRIBE, "topic3"); + } + + @Test + public void givenSubscribeWithDeniedTopic_whenProcessSucceeds_thenPublishSubscribedWithGrantedSubscriptionsOnly() { + when(ctx.getMqttVersion()).thenReturn(MqttVersion.MQTT_5); + when(ctx.getChannel()).thenReturn(mock(ChannelHandlerContext.class)); + SessionInfo sessionInfo = mock(SessionInfo.class); + when(ctx.getSessionInfo()).thenReturn(sessionInfo); + when(sessionInfo.isPersistent()).thenReturn(false); + when(clientSubscriptionService.getClientSubscriptions(any())).thenReturn(Collections.emptySet()); + when(retainedMsgService.getRetainedMessages(any())).thenReturn(Collections.emptyList()); + + when(authorizationRuleService.isSubAuthorized(eq("topic1"), any())).thenReturn(true); + when(authorizationRuleService.isSubAuthorized(eq("topic2"), any())).thenReturn(false); // denied -> excluded from SUBACK grants + when(authorizationRuleService.isSubAuthorized(eq("topic3"), any())).thenReturn(true); + + MqttSubscribeMsg msg = new MqttSubscribeMsg(UUID.randomUUID(), 1, getTopicSubscriptions()); + mqttSubscribeHandler.process(ctx, msg); + + List grantedSubscriptions = List.of(getTopicSubscription("topic1", 0), getTopicSubscription("topic3", 2)); + + // persistence is requested only for the granted subscriptions; drive the success callback to trigger the event + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(BasicCallback.class); + verify(clientSubscriptionService).subscribeAndPersist(any(), eq(grantedSubscriptions), callbackCaptor.capture()); + callbackCaptor.getValue().onSuccess(); + + // only the granted (authorized) subscriptions are reported as CLIENT_SUBSCRIBED; the denied topic2 is excluded + verify(integrationLifecycleEventPublisher).publishSubscribed(ctx, grantedSubscriptions); + } + @Test public void givenMultiLvlRootSub_whenCollectMqttReasonCodes_thenReturnExpectedResult() { when(ctx.getMqttVersion()).thenReturn(MqttVersion.MQTT_5); diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandlerTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandlerTest.java index b228e2c95..fb7296630 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandlerTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/actors/client/service/handlers/MqttUnsubscribeHandlerTest.java @@ -15,15 +15,21 @@ */ package org.thingsboard.mqtt.broker.actors.client.service.handlers; +import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.mqtt.MqttReasonCodes; import io.netty.handler.codec.mqtt.MqttVersion; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.junit.MockitoJUnitRunner; import org.thingsboard.mqtt.broker.actors.client.messages.mqtt.MqttUnsubscribeMsg; import org.thingsboard.mqtt.broker.actors.client.service.subscription.ClientSubscriptionService; +import org.thingsboard.mqtt.broker.common.data.BasicCallback; import org.thingsboard.mqtt.broker.common.data.SessionInfo; +import org.thingsboard.mqtt.broker.common.data.subscription.ClientTopicSubscription; +import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.mqtt.MqttMessageGenerator; import org.thingsboard.mqtt.broker.service.mqtt.persistence.application.ApplicationPersistenceProcessor; import org.thingsboard.mqtt.broker.service.subscription.shared.TopicSharedSubscription; @@ -37,6 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @@ -48,6 +55,7 @@ public class MqttUnsubscribeHandlerTest { MqttMessageGenerator mqttMessageGenerator; ClientSubscriptionService clientSubscriptionService; ApplicationPersistenceProcessor applicationPersistenceProcessor; + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; MqttUnsubscribeHandler mqttUnsubscribeHandler; ClientSessionCtx ctx; @@ -58,7 +66,8 @@ public void setUp() { mqttMessageGenerator = mock(MqttMessageGenerator.class); clientSubscriptionService = mock(ClientSubscriptionService.class); applicationPersistenceProcessor = mock(ApplicationPersistenceProcessor.class); - mqttUnsubscribeHandler = spy(new MqttUnsubscribeHandler(mqttMessageGenerator, clientSubscriptionService, applicationPersistenceProcessor)); + integrationLifecycleEventPublisher = mock(IntegrationLifecycleEventPublisher.class); + mqttUnsubscribeHandler = spy(new MqttUnsubscribeHandler(mqttMessageGenerator, clientSubscriptionService, applicationPersistenceProcessor, integrationLifecycleEventPublisher)); ctx = mock(ClientSessionCtx.class); sessionInfo = mock(SessionInfo.class); @@ -94,6 +103,48 @@ public void testProcess_MQTT3AppClient() { verify(applicationPersistenceProcessor).stopProcessingSharedSubscriptions(any(), eq(Set.of(new TopicSharedSubscription("topic", "group")))); } + @Test + public void givenSubscribedAndNeverSubscribedFilters_whenProcess_thenEmitOnlyRemovedFilters() { + String clientId = "client-1"; + when(ctx.getClientId()).thenReturn(clientId); + when(ctx.getChannel()).thenReturn(mock(ChannelHandlerContext.class)); + lenient().when(clientSubscriptionService.getClientSubscriptions(clientId)) + .thenReturn(Set.of(new ClientTopicSubscription("a/b", 1), new ClientTopicSubscription("c/d", 1))); + + mqttUnsubscribeHandler.process(ctx, new MqttUnsubscribeMsg(UUID.randomUUID(), 1, List.of("a/b", "never/subscribed"))); + + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(BasicCallback.class); + verify(clientSubscriptionService).unsubscribeAndPersist(eq(clientId), eq(List.of("a/b", "never/subscribed")), callbackCaptor.capture()); + callbackCaptor.getValue().onSuccess(); + + // "never/subscribed" was never subscribed; only the actually-removed "a/b" should produce a CLIENT_UNSUBSCRIBED event + verify(integrationLifecycleEventPublisher).publishUnsubscribed(ctx, List.of(new ClientTopicSubscription("a/b", 1))); + } + + @Test + @SuppressWarnings("unchecked") + public void givenSharedSubscription_whenUnsubscribe_thenEmitsSubscriptionWithShareName() { + String clientId = "client-1"; + when(ctx.getClientId()).thenReturn(clientId); + when(ctx.getChannel()).thenReturn(mock(ChannelHandlerContext.class)); + lenient().when(clientSubscriptionService.getClientSubscriptions(clientId)) + .thenReturn(Set.of(new ClientTopicSubscription("topic", 1, "group"))); + + mqttUnsubscribeHandler.process(ctx, new MqttUnsubscribeMsg(UUID.randomUUID(), 1, List.of("$share/group/topic"))); + + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(BasicCallback.class); + verify(clientSubscriptionService).unsubscribeAndPersist(eq(clientId), eq(List.of("$share/group/topic")), callbackCaptor.capture()); + callbackCaptor.getValue().onSuccess(); + + // The removed shared subscription must carry its shareName so $share/group/topic can be reconstructed downstream + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(integrationLifecycleEventPublisher).publishUnsubscribed(eq(ctx), captor.capture()); + List removed = captor.getValue(); + assertEquals(1, removed.size()); + assertEquals("topic", removed.get(0).getTopicFilter()); + assertEquals("group", removed.get(0).getShareName()); + } + @Test public void testCollectUniqueSharedSubscriptions() { List topics = List.of( diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationServiceTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationServiceTest.java index 0dbe11a02..86e421d75 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationServiceTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/DefaultPlatformIntegrationServiceTest.java @@ -93,7 +93,9 @@ void setUp() { integration.setId(integrationId); integration.setName("TestIntegration"); ObjectNode configuration = JacksonUtil.newObjectNode(); - configuration.set("topicFilters", JacksonUtil.newArrayNode()); + ArrayNode topicFilters = JacksonUtil.newArrayNode(); + topicFilters.add("test/topic"); + configuration.set("topicFilters", topicFilters); integration.setConfiguration(configuration); when(eventService.saveAsync(any())).thenReturn(Futures.immediateFuture(null)); diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImplTest.java new file mode 100644 index 000000000..a2c324f67 --- /dev/null +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventPublisherImplTest.java @@ -0,0 +1,361 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import io.netty.handler.codec.mqtt.MqttVersion; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.mqtt.broker.common.data.ClientInfo; +import org.thingsboard.mqtt.broker.common.data.SessionInfo; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.common.data.subscription.SubscriptionOptions; +import org.thingsboard.mqtt.broker.common.data.subscription.TopicSubscription; +import org.thingsboard.mqtt.broker.gen.queue.RetainHandling; +import org.thingsboard.mqtt.broker.gen.queue.TopicSubscriptionProto; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.queue.cluster.ServiceInfoProvider; +import org.thingsboard.mqtt.broker.service.mqtt.persistence.integration.IntegrationEventMsgQueuePublisher; +import org.thingsboard.mqtt.broker.service.processing.PublishMsgCallback; +import org.thingsboard.mqtt.broker.service.stats.DroppedLifecycleEventStats; +import org.thingsboard.mqtt.broker.service.stats.StatsManager; +import org.thingsboard.mqtt.broker.session.ClientSessionCtx; +import org.thingsboard.mqtt.broker.session.DisconnectReasonType; + +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class IntegrationLifecycleEventPublisherImplTest { + + @Mock + private IntegrationLifecycleEventTypeCache lifecycleEventTypeCache; + @Mock + private IntegrationEventMsgQueuePublisher integrationEventMsgQueuePublisher; + @Mock + private StatsManager statsManager; + @Mock + private DroppedLifecycleEventStats droppedLifecycleEventStats; + @Mock + private ServiceInfoProvider serviceInfoProvider; + @Mock + private SessionInfo sessionInfo; + @Mock + private ClientInfo clientInfo; + @Mock + private TopicSubscription topicSubscription; + @Mock + private ClientSessionCtx ctx; + + private IntegrationLifecycleEventPublisherImpl publisher; + + @Before + public void setUp() { + when(statsManager.getDroppedLifecycleEventStats()).thenReturn(droppedLifecycleEventStats); + publisher = new IntegrationLifecycleEventPublisherImpl(lifecycleEventTypeCache, integrationEventMsgQueuePublisher, statsManager, serviceInfoProvider); + publisher.init(); + } + + private void stubCtxSession() { + when(ctx.getSessionInfo()).thenReturn(sessionInfo); + when(ctx.getUsername()).thenReturn("demo"); + when(ctx.getMqttVersion()).thenReturn(MqttVersion.MQTT_5); + when(sessionInfo.getClientInfo()).thenReturn(clientInfo); + when(clientInfo.getClientId()).thenReturn("client-1"); + when(clientInfo.getClientIpAdr()).thenReturn(new byte[]{127, 0, 0, 1}); + when(sessionInfo.getSessionId()).thenReturn(UUID.randomUUID()); + when(sessionInfo.getServiceId()).thenReturn("tbmq-node-1"); + when(sessionInfo.isCleanStart()).thenReturn(true); + when(sessionInfo.getKeepAlive()).thenReturn(60); + when(sessionInfo.safeGetSessionExpiryInterval()).thenReturn(0); + } + + @Test + public void givenSubscriber_whenPublishConnected_thenSetsUsernameProtocolAndExpiry() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + when(ctx.getClientCertCn()).thenReturn("device-42"); + + publisher.publishConnected(ctx); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto proto = captor.getValue().getValue(); + org.junit.Assert.assertEquals("demo", proto.getUsername()); + org.junit.Assert.assertEquals("device-42", proto.getClientCertCn()); + org.junit.Assert.assertEquals(5, proto.getProtocolVersion()); + org.junit.Assert.assertEquals(0L, proto.getSessionExpiryInterval()); + org.junit.Assert.assertTrue(proto.getCleanStart()); + } + + @Test + public void givenNoSubscribers_whenPublishConnected_thenEarlyReturnAndNeverPublishes() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)).thenReturn(Set.of()); + + publisher.publishConnected(ctx); + + verifyNoInteractions(integrationEventMsgQueuePublisher); + verify(droppedLifecycleEventStats, never()).increment(); + } + + @Test + public void givenSubscriber_whenPublishConnected_thenSendsEventMsgDirectly() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + + publisher.publishConnected(ctx); + + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("ie-1"), any(TbProtoQueueMsg.class), any(PublishMsgCallback.class)); + } + + @Test + public void givenSendCallbackFails_whenPublishConnected_thenNoKeyAndIncrementsDroppedMetric() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + + publisher.publishConnected(ctx); + + ArgumentCaptor> msgCaptor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + ArgumentCaptor cbCaptor = ArgumentCaptor.forClass(PublishMsgCallback.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("ie-1"), msgCaptor.capture(), cbCaptor.capture()); + + // no record key: the event stream is single-partition with delete retention, so the key is unused + org.junit.Assert.assertNull(msgCaptor.getValue().getKey()); + + // an asynchronous Kafka send failure must be counted via the callback, not silently swallowed + cbCaptor.getValue().onFailure(new RuntimeException("kafka send failed")); + verify(droppedLifecycleEventStats).increment(); + } + + @Test + public void givenPublisherThrows_whenPublishConnected_thenSwallowsAndIncrementsDroppedMetric() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + doThrow(new RuntimeException("kafka down")) + .when(integrationEventMsgQueuePublisher).sendEventMsg(anyString(), any(), any()); + + // must not throw + publisher.publishConnected(ctx); + + verify(droppedLifecycleEventStats).increment(); + } + + @Test + public void givenCacheThrows_whenPublishDisconnected_thenSwallowsAndIncrementsDroppedMetric() { + doThrow(new RuntimeException("cache boom")) + .when(lifecycleEventTypeCache).getIntegrationIds(ClientLifecycleEventType.CLIENT_DISCONNECTED); + + publisher.publishDisconnected(ctx, DisconnectReasonType.ON_DISCONNECT_MSG); + + verify(droppedLifecycleEventStats).increment(); + } + + @Test + public void givenSubscriber_whenPublishDisconnected_thenSetsMqttStandardReasonCode() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_DISCONNECTED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + + publisher.publishDisconnected(ctx, DisconnectReasonType.ON_CLUSTER_CONFLICTING_SESSIONS); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto p = captor.getValue().getValue(); + org.junit.Assert.assertEquals("CLIENT_DISCONNECTED", p.getEventType()); + // MQTT-standard reason-code name (Netty MqttReasonCodes.Disconnect), not the internal DisconnectReasonType name + org.junit.Assert.assertEquals("SESSION_TAKEN_OVER", p.getDisconnectReason()); + } + + @Test + public void givenSubscriber_whenPublishSubscribed_thenProtoCarriesFullSubscriptionDetails() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("foo/bar"); + when(topicSubscription.getQos()).thenReturn(2); + when(topicSubscription.getShareName()).thenReturn("g1"); + when(topicSubscription.getSubscriptionId()).thenReturn(7); + when(topicSubscription.getOptions()).thenReturn( + new SubscriptionOptions(true, true, SubscriptionOptions.RetainHandlingPolicy.DONT_SEND_AT_SUBSCRIBE)); + + publisher.publishSubscribed(ctx, List.of(topicSubscription)); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + TopicSubscriptionProto sub = captor.getValue().getValue().getSubscriptions(0); + org.junit.Assert.assertEquals("foo/bar", sub.getTopic()); + org.junit.Assert.assertEquals(2, sub.getQos()); + org.junit.Assert.assertEquals("g1", sub.getShareName()); + org.junit.Assert.assertEquals(7, sub.getSubscriptionId()); + org.junit.Assert.assertTrue(sub.getOptions().getNoLocal()); + org.junit.Assert.assertTrue(sub.getOptions().getRetainAsPublish()); + org.junit.Assert.assertEquals(RetainHandling.DONT_SEND, sub.getOptions().getRetainHandling()); + } + + @Test + public void givenSubscriber_whenPublishSubscribed_thenSendsEventMsgPerIntegration() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("test/topic"); + + publisher.publishSubscribed(ctx, List.of(topicSubscription)); + + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("ie-1"), any(TbProtoQueueMsg.class), any(PublishMsgCallback.class)); + verify(droppedLifecycleEventStats, never()).increment(); + } + + @Test + public void givenPublisherThrows_whenPublishSubscribed_thenSwallowsAndIncrementsDroppedMetric() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("test/topic"); + doThrow(new RuntimeException("send failure")) + .when(integrationEventMsgQueuePublisher).sendEventMsg(anyString(), any(), any()); + + publisher.publishSubscribed(ctx, List.of(topicSubscription)); + + verify(droppedLifecycleEventStats).increment(); + } + + @Test + public void givenSubscriber_whenPublishUnsubscribed_thenSendsEventMsgPerIntegration() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("test/topic"); + + publisher.publishUnsubscribed(ctx, List.of(topicSubscription)); + + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("ie-1"), any(TbProtoQueueMsg.class), any(PublishMsgCallback.class)); + verify(droppedLifecycleEventStats, never()).increment(); + } + + @Test + public void givenSharedSubscription_whenPublishUnsubscribed_thenProtoCarriesTopicFilterAndShareName() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("foo/bar"); + when(topicSubscription.getShareName()).thenReturn("g1"); + + publisher.publishUnsubscribed(ctx, List.of(topicSubscription)); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + TopicSubscriptionProto sub = captor.getValue().getValue().getSubscriptions(0); + org.junit.Assert.assertEquals("foo/bar", sub.getTopic()); + org.junit.Assert.assertTrue(sub.hasShareName()); + org.junit.Assert.assertEquals("g1", sub.getShareName()); + // an UNSUBSCRIBE carries only identity — no qos/options/subscriptionId on the wire + org.junit.Assert.assertFalse(sub.hasSubscriptionId()); + } + + @Test + public void givenPublisherThrows_whenPublishUnsubscribed_thenSwallowsAndIncrementsDroppedMetric() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED)).thenReturn(Set.of("ie-1")); + stubCtxSession(); + when(topicSubscription.getTopicFilter()).thenReturn("test/topic"); + doThrow(new RuntimeException("send failure")) + .when(integrationEventMsgQueuePublisher).sendEventMsg(anyString(), any(), any()); + + publisher.publishUnsubscribed(ctx, List.of(topicSubscription)); + + verify(droppedLifecycleEventStats).increment(); + } + + @Test + public void givenSubscriber_whenPublishAuthorizationDenied_thenBuildsDenyEvent() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_AUTHORIZATION_FAILED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + + publisher.publishAuthorizationDenied(ctx, AuthorizationAction.PUBLISH, "zxc/demo/topic"); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto p = captor.getValue().getValue(); + org.junit.Assert.assertEquals("CLIENT_AUTHORIZATION_FAILED", p.getEventType()); + org.junit.Assert.assertEquals("publish", p.getAction()); + org.junit.Assert.assertEquals("zxc/demo/topic", p.getTopic()); + org.junit.Assert.assertEquals("demo", p.getUsername()); + } + + @Test + public void givenSubscriber_whenPublishAuthenticatedFailure_thenBuildsFailureEvent() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_AUTHENTICATION_FAILED)).thenReturn(Set.of("int-1")); + when(serviceInfoProvider.getServiceId()).thenReturn("tbmq-node-1"); + when(ctx.getSessionId()).thenReturn(UUID.randomUUID()); + when(ctx.getAddressBytes()).thenReturn(new byte[]{127, 0, 0, 1}); + when(ctx.getMqttVersion()).thenReturn(MqttVersion.MQTT_5); + when(ctx.getUsername()).thenReturn("demo"); + publisher.publishAuthenticationFailed(ctx, "client-1", "Invalid credentials"); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto p = captor.getValue().getValue(); + org.junit.Assert.assertEquals("CLIENT_AUTHENTICATION_FAILED", p.getEventType()); + org.junit.Assert.assertEquals("client-1", p.getClientId()); + org.junit.Assert.assertEquals("demo", p.getUsername()); + org.junit.Assert.assertEquals("Invalid credentials", p.getReason()); + } + + @Test + public void givenSubscriber_whenPublishConnectionFailed_thenBuildsFailureEvent() { + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTION_FAILED)).thenReturn(Set.of("int-1")); + stubCtxSession(); + + publisher.publishConnectionFailed(ctx, sessionInfo, "QUOTA_EXCEEDED"); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto p = captor.getValue().getValue(); + org.junit.Assert.assertEquals("CLIENT_CONNECTION_FAILED", p.getEventType()); + org.junit.Assert.assertEquals("QUOTA_EXCEEDED", p.getReason()); + org.junit.Assert.assertEquals("client-1", p.getClientId()); + org.junit.Assert.assertEquals("demo", p.getUsername()); + } + + @Test + public void givenPreConnectionValidationFailure_whenPublishConnectionFailed_thenBuildsFromPassedSessionInfo() { + // Pre-connection refusals happen before ctx.setSessionInfo, so the session is passed explicitly. + when(lifecycleEventTypeCache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTION_FAILED)).thenReturn(Set.of("int-1")); + when(ctx.getUsername()).thenReturn("demo"); + when(sessionInfo.getClientInfo()).thenReturn(clientInfo); + when(clientInfo.getClientId()).thenReturn("client-1"); + when(clientInfo.getClientIpAdr()).thenReturn(new byte[]{127, 0, 0, 1}); + when(sessionInfo.getSessionId()).thenReturn(UUID.randomUUID()); + when(sessionInfo.getServiceId()).thenReturn("tbmq-node-1"); + + publisher.publishConnectionFailed(ctx, sessionInfo, "CONNECTION_REFUSED_CLIENT_IDENTIFIER_NOT_VALID"); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(TbProtoQueueMsg.class); + verify(integrationEventMsgQueuePublisher).sendEventMsg(eq("int-1"), captor.capture(), any()); + ClientLifecycleEventMsgProto p = captor.getValue().getValue(); + org.junit.Assert.assertEquals("CLIENT_CONNECTION_FAILED", p.getEventType()); + org.junit.Assert.assertEquals("CONNECTION_REFUSED_CLIENT_IDENTIFIER_NOT_VALID", p.getReason()); + org.junit.Assert.assertEquals("client-1", p.getClientId()); + org.junit.Assert.assertEquals("demo", p.getUsername()); + } +} diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImplTest.java new file mode 100644 index 000000000..8647eff7a --- /dev/null +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/integration/IntegrationLifecycleEventTypeCacheImplTest.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.integration; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class IntegrationLifecycleEventTypeCacheImplTest { + + private IntegrationLifecycleEventTypeCacheImpl cache; + + @Before + public void setUp() { + cache = new IntegrationLifecycleEventTypeCacheImpl(); + } + + @Test + public void givenEmptyCache_whenGetIntegrationIds_thenReturnsEmptyNeverNull() { + Set ids = cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED); + assertTrue(ids.isEmpty()); + } + + @Test + public void givenEmptyCache_whenGetIntegrationIdsTwice_thenSameSharedEmptyInstance() { + Set first = cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED); + Set second = cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_DISCONNECTED); + assertSame(first, second); // allocation-free reads share one empty set + } + + @Test + public void givenPut_whenGetIntegrationIds_thenReverseIndexResolvesIds() { + cache.put("ie-1", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED, ClientLifecycleEventType.CLIENT_SUBSCRIBED)); + cache.put("ie-2", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + + assertEquals(Set.of("ie-1", "ie-2"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)); + assertEquals(Set.of("ie-1"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED)); + assertTrue(cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_UNSUBSCRIBED).isEmpty()); + } + + @Test + public void givenPutThenRemove_whenGetIntegrationIds_thenReverseIndexRebuilt() { + cache.put("ie-1", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + cache.put("ie-2", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + cache.remove("ie-1"); + + assertEquals(Set.of("ie-2"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)); + } + + @Test + public void givenPutEmpty_whenGetIntegrationIds_thenTreatedAsRemoval() { + cache.put("ie-1", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + cache.put("ie-1", Set.of()); // empty == remove + assertTrue(cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED).isEmpty()); + } + + @Test + public void givenResult_whenMutate_thenImmutable() { + cache.put("ie-1", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + Set ids = cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED); + assertThrows(UnsupportedOperationException.class, () -> ids.add("ie-x")); + } + + @Test + public void givenConfigProto_whenProcess_thenReverseIndexResolvesIds() { + IntegrationLifecycleConfigProto proto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("ie-1") + .addLifecycleEventTypes("CLIENT_CONNECTED") + .addLifecycleEventTypes("CLIENT_SUBSCRIBED") + .build(); + + cache.processIntegrationLifecycleConfig(proto); + + assertEquals(Set.of("ie-1"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)); + assertEquals(Set.of("ie-1"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_SUBSCRIBED)); + } + + @Test + public void givenDeletedConfigProto_whenProcess_thenRemoved() { + cache.put("ie-1", Set.of(ClientLifecycleEventType.CLIENT_CONNECTED)); + IntegrationLifecycleConfigProto proto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("ie-1") + .setDeleted(true) + .build(); + + cache.processIntegrationLifecycleConfig(proto); + + assertTrue(cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED).isEmpty()); + } + + @Test + public void givenConfigProtoWithUnknownType_whenProcess_thenKnownTypesApplied() { + IntegrationLifecycleConfigProto proto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("ie-1") + .addLifecycleEventTypes("CLIENT_CONNECTED") + .addLifecycleEventTypes("NOT_A_REAL_TYPE") + .build(); + + cache.processIntegrationLifecycleConfig(proto); + + assertEquals(Set.of("ie-1"), cache.getIntegrationIds(ClientLifecycleEventType.CLIENT_CONNECTED)); + } + +} diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImplTest.java new file mode 100644 index 000000000..25d25d9fc --- /dev/null +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/persistence/integration/IntegrationEventMsgQueuePublisherImplTest.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.mqtt.persistence.integration; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.queue.TbQueueCallback; +import org.thingsboard.mqtt.broker.queue.TbQueueProducer; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.queue.provider.integration.IntegrationMsgQueueProvider; +import org.thingsboard.mqtt.broker.service.analysis.ClientLogger; +import org.thingsboard.mqtt.broker.service.processing.PublishMsgCallback; +import org.thingsboard.mqtt.broker.service.util.IntegrationHelperService; + +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class IntegrationEventMsgQueuePublisherImplTest { + + @Mock + private ClientLogger clientLogger; + @Mock + private IntegrationMsgQueueProvider msgQueueProvider; + @Mock + private IntegrationHelperService integrationHelperService; + @Mock + private TbQueueProducer> producer; + + private IntegrationEventMsgQueuePublisherImpl publisher; + + private static final String INTEGRATION_ID = "ie-1"; + private static final String EVENT_TOPIC = "tbmq.ie.event.ie_1"; + + @Before + public void setUp() { + when(msgQueueProvider.getIeEventMsgProducer()).thenReturn(producer); + publisher = new IntegrationEventMsgQueuePublisherImpl(clientLogger, msgQueueProvider, integrationHelperService); + publisher.init(); + } + + @Test + public void givenEventMsg_whenSendEventMsg_thenForwardsToEventTopicOnPartitionZero() { + when(integrationHelperService.getIntegrationEventTopic(INTEGRATION_ID)).thenReturn(EVENT_TOPIC); + TbProtoQueueMsg queueMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), ClientLifecycleEventMsgProto.newBuilder().build()); + + publisher.sendEventMsg(INTEGRATION_ID, queueMsg, PublishMsgCallback.EMPTY); + + verify(producer).send(eq(EVENT_TOPIC), eq(0), eq(queueMsg), any(TbQueueCallback.class)); + } + +} diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/validation/PublishMsgValidationServiceImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/validation/PublishMsgValidationServiceImplTest.java index b484fd3f2..7da182eff 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/validation/PublishMsgValidationServiceImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/mqtt/validation/PublishMsgValidationServiceImplTest.java @@ -19,18 +19,21 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.mqtt.broker.dao.service.DefaultTopicValidationService; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRuleService; +import org.thingsboard.mqtt.broker.service.integration.AuthorizationAction; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventPublisher; import org.thingsboard.mqtt.broker.service.mqtt.PublishMsg; import org.thingsboard.mqtt.broker.session.ClientSessionCtx; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -39,12 +42,14 @@ @ContextConfiguration(classes = PublishMsgValidationServiceImpl.class) public class PublishMsgValidationServiceImplTest { - @MockBean + @MockitoBean DefaultTopicValidationService topicValidationService; - @MockBean + @MockitoBean AuthorizationRuleService authorizationRuleService; + @MockitoBean + IntegrationLifecycleEventPublisher integrationLifecycleEventPublisher; - @SpyBean + @MockitoSpyBean PublishMsgValidationServiceImpl publishMsgValidationService; ClientSessionCtx ctx; @@ -77,17 +82,20 @@ public void givenPubMsg_whenValidatePubMsg_thenTopicAndAuthChecked() { } @Test - public void givenClientContextAndAllowPublishToTopic_whenValidateClientAccess_thenSuccess() { + public void givenClientContextAndAllowPublishToTopic_whenValidateClientAccess_thenSuccessAndNoAuthorizationDeniedEvent() { when(authorizationRuleService.isPubAuthorized(any(), any(), any())).thenReturn(true); boolean result = publishMsgValidationService.validateClientAccess(ctx, "clientId", "topic/1"); Assert.assertTrue(result); + verify(integrationLifecycleEventPublisher, never()).publishAuthorizationDenied(any(), any(), any()); } @Test - public void givenClientContextAndNotAllowPublishToTopic_whenValidateClientAccess_thenFailure() { + public void givenClientContextAndNotAllowPublishToTopic_whenValidateClientAccess_thenFailureAndAuthorizationDeniedEventPublished() { when(authorizationRuleService.isPubAuthorized(any(), any(), any())).thenReturn(false); boolean result = publishMsgValidationService.validateClientAccess(ctx, "clientId", "topic/1"); Assert.assertFalse(result); + verify(integrationLifecycleEventPublisher, times(1)) + .publishAuthorizationDenied(eq(ctx), eq(AuthorizationAction.PUBLISH), eq("topic/1")); } } diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsConsumerImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsConsumerImplTest.java index 6980f09e9..67898bafd 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsConsumerImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsConsumerImplTest.java @@ -23,7 +23,9 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; import org.thingsboard.mqtt.broker.gen.queue.ClientSessionStatsCleanupProto; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; import org.thingsboard.mqtt.broker.gen.queue.InternodeNotificationProto; import org.thingsboard.mqtt.broker.gen.queue.MqttAuthProviderProto; import org.thingsboard.mqtt.broker.gen.queue.MqttAuthSettingsProto; @@ -33,6 +35,7 @@ import org.thingsboard.mqtt.broker.queue.provider.InternodeNotificationsQueueFactory; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRoutingService; import org.thingsboard.mqtt.broker.service.auth.providers.MqttAuthProviderNotificationManager; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventTypeCache; import org.thingsboard.mqtt.broker.service.mqtt.client.session.ClientSessionStatsCleanupProcessor; import java.util.List; @@ -65,6 +68,9 @@ public class InternodeNotificationsConsumerImplTest { @Mock private AuthorizationRoutingService authorizationRoutingService; + @Mock + private IntegrationLifecycleEventTypeCache integrationLifecycleEventTypeCache; + @Mock private TbQueueConsumer> consumer; @@ -78,7 +84,8 @@ public void setUp() { serviceInfoProvider, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor, - authorizationRoutingService); + authorizationRoutingService, + integrationLifecycleEventTypeCache); ReflectionTestUtils.setField(notificationsConsumer, "pollDuration", 1L); @@ -157,6 +164,60 @@ public void testStartConsuming_AndProcessesMessageWithClientSessionStatsCleanupR verify(consumer).unsubscribeAndClose(); } + @Test + public void testStartConsuming_AndProcessesMessageWithIntegrationLifecycleConfig() { + IntegrationLifecycleConfigProto configProto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("integration-1") + .addLifecycleEventTypes(ClientLifecycleEventType.CLIENT_CONNECTED.name()) + .build(); + InternodeNotificationProto proto = InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(configProto) + .build(); + + TbProtoQueueMsg msg = new TbProtoQueueMsg<>("nodeA", proto); + + when(consumer.poll(anyLong())) + .thenReturn(List.of(msg)) + .thenReturn(List.of()); + + notificationsConsumer.startConsuming(); + + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> + verify(integrationLifecycleEventTypeCache).processIntegrationLifecycleConfig(configProto)); + + verifyNoInteractions(authorizationRoutingService, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor); + + notificationsConsumer.destroy(); + verify(consumer).unsubscribeAndClose(); + } + + @Test + public void testStartConsuming_AndProcessesMessageWithIntegrationLifecycleConfigDeleted() { + IntegrationLifecycleConfigProto configProto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("integration-1") + .setDeleted(true) + .build(); + InternodeNotificationProto proto = InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(configProto) + .build(); + + TbProtoQueueMsg msg = new TbProtoQueueMsg<>("nodeA", proto); + + when(consumer.poll(anyLong())) + .thenReturn(List.of(msg)) + .thenReturn(List.of()); + + notificationsConsumer.startConsuming(); + + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> + verify(integrationLifecycleEventTypeCache).processIntegrationLifecycleConfig(configProto)); + + verifyNoInteractions(authorizationRoutingService, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor); + + notificationsConsumer.destroy(); + verify(consumer).unsubscribeAndClose(); + } + @Test public void testDestroy_ClosesResources() { when(consumer.poll(anyLong())).thenReturn(List.of()); diff --git a/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsServiceImplTest.java b/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsServiceImplTest.java index 1f579e99d..5f139dede 100644 --- a/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsServiceImplTest.java +++ b/application/src/test/java/org/thingsboard/mqtt/broker/service/notification/InternodeNotificationsServiceImplTest.java @@ -21,7 +21,9 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; import org.thingsboard.mqtt.broker.gen.queue.ClientSessionStatsCleanupProto; +import org.thingsboard.mqtt.broker.gen.queue.IntegrationLifecycleConfigProto; import org.thingsboard.mqtt.broker.gen.queue.InternodeNotificationProto; import org.thingsboard.mqtt.broker.gen.queue.MqttAuthProviderProto; import org.thingsboard.mqtt.broker.gen.queue.MqttAuthSettingsProto; @@ -32,6 +34,7 @@ import org.thingsboard.mqtt.broker.queue.provider.InternodeNotificationsQueueFactory; import org.thingsboard.mqtt.broker.service.auth.AuthorizationRoutingService; import org.thingsboard.mqtt.broker.service.auth.providers.MqttAuthProviderNotificationManager; +import org.thingsboard.mqtt.broker.service.integration.IntegrationLifecycleEventTypeCache; import org.thingsboard.mqtt.broker.service.mqtt.client.session.ClientSessionStatsCleanupProcessor; import java.util.List; @@ -65,6 +68,9 @@ public class InternodeNotificationsServiceImplTest { @Mock private AuthorizationRoutingService authorizationRoutingService; + @Mock + private IntegrationLifecycleEventTypeCache integrationLifecycleEventTypeCache; + @Mock private TbQueueProducer> producer; @@ -81,7 +87,8 @@ public void setUp() { helper, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor, - authorizationRoutingService + authorizationRoutingService, + integrationLifecycleEventTypeCache ); service.init(); } @@ -203,6 +210,42 @@ public void testBroadcast_ToSelf_WithClientSessionStartCleanupRequest() { verifyNoInteractions(authorizationRoutingService, mqttClientAuthProviderManager, producer); } + @Test + public void testBroadcast_ToSelf_WithIntegrationLifecycleConfig() { + IntegrationLifecycleConfigProto configProto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("integration-1") + .addLifecycleEventTypes(ClientLifecycleEventType.CLIENT_CONNECTED.name()) + .build(); + InternodeNotificationProto proto = InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(configProto) + .build(); + + when(helper.getServiceIds()).thenReturn(List.of("nodeA")); + + service.broadcast(proto); + + verify(integrationLifecycleEventTypeCache).processIntegrationLifecycleConfig(configProto); + verifyNoInteractions(authorizationRoutingService, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor, producer); + } + + @Test + public void testBroadcast_ToSelf_WithIntegrationLifecycleConfigDeleted() { + IntegrationLifecycleConfigProto configProto = IntegrationLifecycleConfigProto.newBuilder() + .setIntegrationId("integration-1") + .setDeleted(true) + .build(); + InternodeNotificationProto proto = InternodeNotificationProto.newBuilder() + .setIntegrationLifecycleConfigProto(configProto) + .build(); + + when(helper.getServiceIds()).thenReturn(List.of("nodeA")); + + service.broadcast(proto); + + verify(integrationLifecycleEventTypeCache).processIntegrationLifecycleConfig(configProto); + verifyNoInteractions(authorizationRoutingService, mqttClientAuthProviderManager, clientSessionStatsCleanupProcessor, producer); + } + @Test public void testDestroy_ShouldStopProducer() { service.destroy(); diff --git a/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventType.java b/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventType.java new file mode 100644 index 000000000..99561529b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventType.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.common.data.integration; + +public enum ClientLifecycleEventType { + CLIENT_CONNECTED, + CLIENT_DISCONNECTED, + CLIENT_SUBSCRIBED, + CLIENT_UNSUBSCRIBED, + CLIENT_AUTHENTICATION_FAILED, + CLIENT_AUTHORIZATION_FAILED, + CLIENT_CONNECTION_FAILED +} diff --git a/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventTypeUtil.java b/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventTypeUtil.java new file mode 100644 index 000000000..c7da9872a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/mqtt/broker/common/data/integration/ClientLifecycleEventTypeUtil.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.common.data.integration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class ClientLifecycleEventTypeUtil { + + /** + * Integration configuration JSON key holding the opted-in lifecycle event types. + */ + public static final String LIFECYCLE_EVENT_TYPES_KEY = "lifecycleEventTypes"; + + /** + * Parses event type names into {@link ClientLifecycleEventType} values, invoking {@code onUnknown} + * for every name that does not map to a known type. The callback decides how to react to an unknown + * value (e.g. log a warning and skip it, or throw a validation exception). + */ + public static Set parse(Iterable names, Consumer onUnknown) { + Set eventTypes = new HashSet<>(); + if (names == null) { + return eventTypes; + } + for (String name : names) { + try { + eventTypes.add(ClientLifecycleEventType.valueOf(name)); + } catch (IllegalArgumentException e) { + onUnknown.accept(name); + } + } + return eventTypes; + } + + /** + * Parses a JSON array of event type names. See {@link #parse(Iterable, Consumer)}. + */ + public static Set parse(JsonNode arrayNode, Consumer onUnknown) { + if (arrayNode == null) { + return new HashSet<>(); + } + List names = new ArrayList<>(); + arrayNode.forEach(node -> names.add(node.asText())); + return parse(names, onUnknown); + } + + /** + * Returns the {@link ClientLifecycleEventType} matching the given name, or {@code null} if the name does not + * map to a known type (e.g. an event produced by a newer node). Lets callers handle unknown types gracefully + * instead of throwing. + */ + public static ClientLifecycleEventType fromName(String name) { + if (name == null) { + return null; + } + try { + return ClientLifecycleEventType.valueOf(name); + } catch (IllegalArgumentException e) { + return null; + } + } +} diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/DefaultIntegrationStatisticsService.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/DefaultIntegrationStatisticsService.java index f01418de0..8a56ba522 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/DefaultIntegrationStatisticsService.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/DefaultIntegrationStatisticsService.java @@ -59,6 +59,7 @@ public class DefaultIntegrationStatisticsService implements IntegrationStatistic private static final String STATS_KEY_GAUGE = StatsType.INTEGRATION.getPrintName() + "_stats_gauge"; private final Map managedIntegrationProcessorStats = new ConcurrentHashMap<>(); + private final Map managedEventProcessorStats = new ConcurrentHashMap<>(); private final Map counters = new ConcurrentHashMap<>(); private final Map gauges = new ConcurrentHashMap<>(); private final List managedStats = new CopyOnWriteArrayList<>(); @@ -85,7 +86,7 @@ public void shutdown() { @Override public IntegrationProcessorStats createIntegrationProcessorStats(UUID integrationId) { log.trace("Creating IntegrationProcessorStats, integrationId - {}", integrationId); - IntegrationProcessorStats stats = new IntegrationProcessorStatsImpl(integrationId, statsFactory); + IntegrationProcessorStats stats = new IntegrationProcessorStatsImpl(integrationId, statsFactory, StatsType.INTEGRATION_PROCESSOR); managedIntegrationProcessorStats.put(integrationId, stats); return stats; } @@ -93,7 +94,25 @@ public IntegrationProcessorStats createIntegrationProcessorStats(UUID integratio @Override public void clearIntegrationProcessorStats(UUID integrationId) { log.trace("Clearing IntegrationProcessorStats, integrationId - {}", integrationId); - IntegrationProcessorStats stats = managedIntegrationProcessorStats.get(integrationId); + disable(managedIntegrationProcessorStats, integrationId); + } + + @Override + public IntegrationProcessorStats createIntegrationEventProcessorStats(UUID integrationId) { + log.trace("Creating IntegrationEventProcessorStats, integrationId - {}", integrationId); + IntegrationProcessorStats stats = new IntegrationProcessorStatsImpl(integrationId, statsFactory, StatsType.INTEGRATION_EVENT_PROCESSOR); + managedEventProcessorStats.put(integrationId, stats); + return stats; + } + + @Override + public void clearIntegrationEventProcessorStats(UUID integrationId) { + log.trace("Clearing IntegrationEventProcessorStats, integrationId - {}", integrationId); + disable(managedEventProcessorStats, integrationId); + } + + private static void disable(Map managed, UUID integrationId) { + IntegrationProcessorStats stats = managed.get(integrationId); if (stats != null && stats.isActive()) { stats.disable(); } @@ -172,14 +191,19 @@ public void printStats() { log.info("[{}] Integration Uplink Queue Stats: {}", stats.getName(), statsStr); stats.reset(); } - for (IntegrationProcessorStats stats : new ArrayList<>(managedIntegrationProcessorStats.values())) { + printProcessorStats(managedIntegrationProcessorStats, StatsType.INTEGRATION_PROCESSOR.getPrintName(), "Integration Message Processing Stats"); + printProcessorStats(managedEventProcessorStats, StatsType.INTEGRATION_EVENT_PROCESSOR.getPrintName(), "Integration Event Processing Stats"); + } + + private void printProcessorStats(Map managed, String printName, String label) { + for (IntegrationProcessorStats stats : new ArrayList<>(managed.values())) { String msgStatsStr = stats.getStatsCounters().stream() .map(statsCounter -> statsCounter.getName() + " = [" + statsCounter.get() + "]") .collect(Collectors.joining(" ")); - log.info("[{}][{}] Integration Message Processing Stats: {}", StatsType.INTEGRATION_PROCESSOR.getPrintName(), stats.getIntegrationUuid(), msgStatsStr); + log.info("[{}][{}] {}: {}", printName, stats.getIntegrationUuid(), label, msgStatsStr); if (!stats.isActive()) { log.trace("[{}] Clearing inactive Integration stats", stats.getIntegrationUuid()); - managedIntegrationProcessorStats.computeIfPresent(stats.getIntegrationUuid(), (clientId, oldStats) -> oldStats.isActive() ? oldStats : null); + managed.computeIfPresent(stats.getIntegrationUuid(), (clientId, oldStats) -> oldStats.isActive() ? oldStats : null); } else { stats.reset(); } diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicService.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicService.java index e0518ee0c..dc910681c 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicService.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicService.java @@ -27,4 +27,10 @@ public interface IntegrationTopicService { String getConsumerGroup(String integrationId); + String createEventTopic(String integrationId); + + void deleteEventTopic(String integrationId, BasicCallback callback); + + String getEventConsumerGroup(String integrationId); + } diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImpl.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImpl.java index 528199a0e..6bbb184a8 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImpl.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImpl.java @@ -66,4 +66,36 @@ public String getConsumerGroup(String integrationId) { return integrationHelperService.getIntegrationConsumerGroup(integrationId); } + @Override + public String createEventTopic(String integrationId) { + log.debug("[{}] Creating IE event topic", integrationId); + String eventTopic = integrationHelperService.getIntegrationEventTopic(integrationId); + queueAdmin.createTopic(eventTopic, integrationMsgQueueProvider.getIeEventMsgTopicConfigs()); + return eventTopic; + } + + @Override + public void deleteEventTopic(String integrationId, BasicCallback callback) { + log.debug("[{}] Deleting IE event topic", integrationId); + deleteEventConsumerGroup(integrationId); + String eventTopic = integrationHelperService.getIntegrationEventTopic(integrationId); + queueAdmin.deleteTopic(eventTopic, callback); + } + + @Override + public String getEventConsumerGroup(String integrationId) { + return integrationHelperService.getIntegrationEventConsumerGroup(integrationId); + } + + private void deleteEventConsumerGroup(String integrationId) { + String consumerGroup = getEventConsumerGroup(integrationId); + try { + queueAdmin.deleteConsumerGroup(consumerGroup); + } catch (Exception e) { + if (!(e.getCause() instanceof GroupIdNotFoundException)) { + throw new RuntimeException(e); + } + } + } + } diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperService.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperService.java index 164dedb35..2cd126523 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperService.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperService.java @@ -21,4 +21,8 @@ public interface IntegrationHelperService { String getIntegrationConsumerGroup(String integrationId); + String getIntegrationEventTopic(String integrationId); + + String getIntegrationEventConsumerGroup(String integrationId); + } diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperServiceImpl.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperServiceImpl.java index 6a77d388c..706b18430 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperServiceImpl.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationHelperServiceImpl.java @@ -35,4 +35,14 @@ public String getIntegrationTopic(String integrationId) { public String getIntegrationConsumerGroup(String integrationId) { return kafkaPrefix + IntegrationUtil.getIntegrationConsumerGroup(integrationId); } + + @Override + public String getIntegrationEventTopic(String integrationId) { + return kafkaPrefix + IntegrationUtil.getIntegrationEventTopic(integrationId); + } + + @Override + public String getIntegrationEventConsumerGroup(String integrationId) { + return kafkaPrefix + IntegrationUtil.getIntegrationEventConsumerGroup(integrationId); + } } diff --git a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtil.java b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtil.java index 99e9d2631..9af6c6e8c 100644 --- a/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtil.java +++ b/common/integration/cluster-integration-api/src/main/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtil.java @@ -30,4 +30,15 @@ public static String getIntegrationConsumerGroup(String integrationId) { return IE_CONSUMER_GROUP_PREFIX + UUIDUtil.strUuidReplaceHyphen(integrationId); } + private static final String IE_EVENT_MSG_TOPIC_PREFIX = "tbmq.ie.event."; + private static final String IE_EVENT_CONSUMER_GROUP_PREFIX = "ie-event-consumer-group-"; + + public static String getIntegrationEventTopic(String integrationId) { + return IE_EVENT_MSG_TOPIC_PREFIX + UUIDUtil.strUuidReplaceHyphen(integrationId); + } + + public static String getIntegrationEventConsumerGroup(String integrationId) { + return IE_EVENT_CONSUMER_GROUP_PREFIX + UUIDUtil.strUuidReplaceHyphen(integrationId); + } + } diff --git a/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImplTest.java b/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImplTest.java new file mode 100644 index 000000000..d51c1d831 --- /dev/null +++ b/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/queue/IntegrationTopicServiceImplTest.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.queue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.mqtt.broker.common.data.BasicCallback; +import org.thingsboard.mqtt.broker.queue.TbQueueAdmin; +import org.thingsboard.mqtt.broker.queue.provider.integration.IntegrationMsgQueueProvider; +import org.thingsboard.mqtt.broker.service.util.IntegrationHelperService; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IntegrationTopicServiceImplTest { + + static final String INTEGRATION_ID = "0198e1a0-1111-2222-3333-444455556666"; + static final String EVENT_TOPIC = "tbmq.ie.event." + "0198e1a0111122223333444455556666"; + static final String EVENT_GROUP = "ie-event-consumer-group-" + "0198e1a0111122223333444455556666"; + + @Mock + TbQueueAdmin queueAdmin; + @Mock + IntegrationMsgQueueProvider integrationMsgQueueProvider; + @Mock + IntegrationHelperService integrationHelperService; + + @InjectMocks + IntegrationTopicServiceImpl service; + + @Test + void givenIntegrationId_whenCreateEventTopic_thenCreatesTopicWithEventConfigs() { + Map cfg = Map.of("retention.ms", "60000"); + when(integrationHelperService.getIntegrationEventTopic(INTEGRATION_ID)).thenReturn(EVENT_TOPIC); + when(integrationMsgQueueProvider.getIeEventMsgTopicConfigs()).thenReturn(cfg); + + String topic = service.createEventTopic(INTEGRATION_ID); + + assertThat(topic).isEqualTo(EVENT_TOPIC); + verify(queueAdmin).createTopic(EVENT_TOPIC, cfg); + } + + @Test + void givenIntegrationId_whenDeleteEventTopic_thenDeletesGroupAndTopic() throws Exception { + when(integrationHelperService.getIntegrationEventConsumerGroup(INTEGRATION_ID)).thenReturn(EVENT_GROUP); + when(integrationHelperService.getIntegrationEventTopic(INTEGRATION_ID)).thenReturn(EVENT_TOPIC); + BasicCallback cb = mock(BasicCallback.class); + + service.deleteEventTopic(INTEGRATION_ID, cb); + + verify(queueAdmin).deleteConsumerGroup(EVENT_GROUP); + verify(queueAdmin).deleteTopic(EVENT_TOPIC, cb); + } + +} diff --git a/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtilTest.java b/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtilTest.java new file mode 100644 index 000000000..5eee0f2b3 --- /dev/null +++ b/common/integration/cluster-integration-api/src/test/java/org/thingsboard/mqtt/broker/service/util/IntegrationUtilTest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.service.util; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IntegrationUtilTest { + + private static final String ID = "1d2f5d40-1111-2222-3333-444455556666"; + + @Test + void givenIntegrationId_whenGetIntegrationEventTopic_thenPrefixedAndHyphensRemoved() { + assertThat(IntegrationUtil.getIntegrationEventTopic(ID)) + .isEqualTo("tbmq.ie.event.1d2f5d40111122223333444455556666"); + } + + @Test + void givenIntegrationId_whenGetIntegrationEventConsumerGroup_thenPrefixedAndHyphensRemoved() { + assertThat(IntegrationUtil.getIntegrationEventConsumerGroup(ID)) + .isEqualTo("ie-event-consumer-group-1d2f5d40111122223333444455556666"); + } +} diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegration.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegration.java index f5c41c069..5b86821a7 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegration.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegration.java @@ -16,17 +16,24 @@ package org.thingsboard.mqtt.broker.integration.api; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.mqtt.broker.common.data.event.ErrorEvent; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventType; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; import org.thingsboard.mqtt.broker.common.data.integration.ComponentLifecycleEvent; import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; import org.thingsboard.mqtt.broker.common.data.util.StringUtils; import org.thingsboard.mqtt.broker.common.util.JacksonUtil; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.gen.queue.PublishMsgProto; +import org.thingsboard.mqtt.broker.gen.queue.SubscriptionOptionsProto; +import org.thingsboard.mqtt.broker.gen.queue.TopicSubscriptionProto; +import org.thingsboard.mqtt.broker.integration.api.callback.IntegrationMsgCallback; import org.thingsboard.mqtt.broker.integration.api.data.ContentType; import org.thingsboard.mqtt.broker.integration.api.data.UplinkMetaData; import org.thingsboard.mqtt.broker.integration.api.util.ExceptionUtil; @@ -105,7 +112,16 @@ public void validateConfiguration(IntegrationLifecycleMsg lifecycleMsg, boolean if (lifecycleMsg == null || lifecycleMsg.getConfiguration() == null) { throw new IllegalArgumentException("Integration configuration is empty!"); } - doValidateConfiguration(lifecycleMsg.getConfiguration().get("clientConfiguration"), allowLocalNetworkHosts); + JsonNode clientConfiguration = lifecycleMsg.getConfiguration().get("clientConfiguration"); + doValidateConfiguration(clientConfiguration, allowLocalNetworkHosts); + if (isLifecycleEventsEnabled(lifecycleMsg.getConfiguration())) { + doValidateLifecycleEventsDelivery(clientConfiguration); + } + } + + private static boolean isLifecycleEventsEnabled(JsonNode configuration) { + JsonNode types = configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY); + return types != null && !types.isEmpty(); } @Override @@ -181,6 +197,107 @@ protected ObjectNode constructBody(PublishIntegrationMsgProto msg) { return request; } + @Override + public void processLifecycleEvent(ClientLifecycleEventMsgProto msg, IntegrationMsgCallback callback) { + try { + ObjectNode body = constructLifecycleEventBody(msg); + doProcessLifecycleEvent(body, callback); + } catch (Exception e) { + handleMsgProcessingFailure(e); + callback.onFailure(e); + } + } + + // Receives the event as an ObjectNode (not a pre-serialized String) so JSON-native transports like HTTP can send + // it as application/json, mirroring the message path where constructBody(...) is passed on directly. Transports + // that need bytes on the wire (Kafka, MQTT) serialize it themselves via JacksonUtil.toString(body). + protected void doProcessLifecycleEvent(ObjectNode body, IntegrationMsgCallback callback) { + log.debug("[{}][{}] Lifecycle event processing is not supported by this integration type", getId(), getName()); + callback.onSuccess(); + } + + protected ObjectNode constructLifecycleEventBody(ClientLifecycleEventMsgProto msg) { + ObjectNode body = JacksonUtil.newObjectNode(); + // eventType is the discriminator and is always present; the other string fields are omitted when empty + // (e.g. no username when authentication is disabled, no ipAddress for address-less sessions) so the JSON + // carries only the keys that actually apply rather than default placeholders like "". + body.put("eventType", msg.getEventType()); + putIfNotEmpty(body, "clientId", msg.getClientId()); + putIfNotEmpty(body, "sessionId", msg.getSessionId()); + putIfNotEmpty(body, "ipAddress", msg.getIpAddress()); + body.put("ts", msg.getTs()); + putIfNotEmpty(body, "tbmqNode", msg.getTbmqNode()); + putIfNotEmpty(body, "username", msg.getUsername()); + putIfNotEmpty(body, "clientCertCn", msg.getClientCertCn()); + + // Switch on the canonical enum (parsed leniently) rather than raw string literals so the proto eventType + // and the enum names stay coupled; an unknown/newer type just yields the common fields above. + ClientLifecycleEventType eventType = ClientLifecycleEventTypeUtil.fromName(msg.getEventType()); + if (eventType != null) { + switch (eventType) { + case CLIENT_CONNECTED: + body.put("cleanStart", msg.getCleanStart()); + body.put("keepAlive", msg.getKeepAlive()); + body.put("protocolVersion", msg.getProtocolVersion()); + body.put("sessionExpiryInterval", msg.getSessionExpiryInterval()); + break; + case CLIENT_DISCONNECTED: + putIfNotEmpty(body, "disconnectReason", msg.getDisconnectReason()); + break; + case CLIENT_SUBSCRIBED: + ArrayNode subs = body.putArray("subscriptions"); + for (TopicSubscriptionProto sub : msg.getSubscriptionsList()) { + ObjectNode subNode = subs.addObject() + .put("topicFilter", sub.getTopic()) + .put("qos", sub.getQos()); + if (sub.hasShareName()) { + subNode.put("shareName", sub.getShareName()); + } + if (sub.hasSubscriptionId()) { + subNode.put("subscriptionId", sub.getSubscriptionId()); + } + SubscriptionOptionsProto options = sub.getOptions(); + subNode.putObject("options") + .put("noLocal", options.getNoLocal()) + .put("retainAsPublish", options.getRetainAsPublish()) + .put("retainHandling", options.getRetainHandling().name()); + } + break; + case CLIENT_UNSUBSCRIBED: + ArrayNode unsubscribed = body.putArray("subscriptions"); + for (TopicSubscriptionProto sub : msg.getSubscriptionsList()) { + ObjectNode subNode = unsubscribed.addObject().put("topicFilter", sub.getTopic()); + if (sub.hasShareName()) { + subNode.put("shareName", sub.getShareName()); + } + } + break; + case CLIENT_AUTHENTICATION_FAILED: + // protocolVersion is known by the time authentication runs (the CONNECT variable header is + // parsed first), so emit it like CLIENT_CONNECTED - it is useful context for version-specific + // auth failures (e.g. MQTT 5 enhanced-auth vs 3.1.1 basic). + body.put("protocolVersion", msg.getProtocolVersion()); + putIfNotEmpty(body, "reason", msg.getReason()); + break; + case CLIENT_AUTHORIZATION_FAILED: + putIfNotEmpty(body, "action", msg.getAction()); + putIfNotEmpty(body, "topic", msg.getTopic()); + break; + case CLIENT_CONNECTION_FAILED: + putIfNotEmpty(body, "reason", msg.getReason()); + break; + } + } + body.set("metadata", JacksonUtil.valueToTree(metadataTemplate.getKvMap())); + return body; + } + + private static void putIfNotEmpty(ObjectNode body, String field, String value) { + if (StringUtils.isNotEmpty(value)) { + body.put(field, value); + } + } + protected void handleMsgProcessingFailure(Throwable throwable) { integrationStatistics.incErrorsOccurred(); context.saveErrorEvent(getErrorEvent(throwable)); @@ -244,6 +361,16 @@ protected void doValidateConfiguration(JsonNode clientConfiguration, boolean all } + /** + * Validates that this integration can deliver lifecycle events with the given client configuration. Called only + * when the integration is opted in for lifecycle events. Lifecycle events have no originating MQTT message, so a + * transport with a dynamic (per-message) destination must have a statically configured destination to fall back + * to here. Default is a no-op for transports whose destination is always static (e.g. HTTP URL, Kafka topic). + */ + protected void doValidateLifecycleEventsDelivery(JsonNode clientConfiguration) throws ThingsboardException { + + } + protected void doCheckConnection(Integration integration, IntegrationContext ctx) throws ThingsboardException { } diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/IntegrationStatisticsService.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/IntegrationStatisticsService.java index 16e4111b3..a55f26fec 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/IntegrationStatisticsService.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/IntegrationStatisticsService.java @@ -28,6 +28,10 @@ public interface IntegrationStatisticsService { void clearIntegrationProcessorStats(UUID integrationId); + IntegrationProcessorStats createIntegrationEventProcessorStats(UUID integrationId); + + void clearIntegrationEventProcessorStats(UUID integrationId); + MessagesStats createIeUplinkPublishStats(); void onIntegrationStateUpdate(IntegrationType integrationType, ComponentLifecycleEvent state, boolean success); diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/TbPlatformIntegration.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/TbPlatformIntegration.java index 3bf000382..9796a42f8 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/TbPlatformIntegration.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/TbPlatformIntegration.java @@ -18,6 +18,7 @@ import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.integration.api.callback.IntegrationMsgCallback; @@ -41,6 +42,8 @@ public interface TbPlatformIntegration { void process(PublishIntegrationMsgProto msg, IntegrationMsgCallback callback); + void processLifecycleEvent(ClientLifecycleEventMsgProto msg, IntegrationMsgCallback callback); + IntegrationStatistics popStatistics(); String getIntegrationId(); diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingContext.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingContext.java index 0085035d2..ac7f952d2 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingContext.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingContext.java @@ -17,7 +17,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -26,17 +25,17 @@ import java.util.concurrent.TimeUnit; @Slf4j -public class IntegrationPackProcessingContext { +public class IntegrationPackProcessingContext { private final String integrationId; @Getter - private final ConcurrentMap pendingMap; + private final ConcurrentMap pendingMap; @Getter - private final ConcurrentMap failedMap = new ConcurrentHashMap<>(); + private final ConcurrentMap failedMap = new ConcurrentHashMap<>(); private final CountDownLatch processingTimeoutLatch; - public IntegrationPackProcessingContext(String integrationId, ConcurrentMap pendingMessages) { + public IntegrationPackProcessingContext(String integrationId, ConcurrentMap pendingMessages) { this.integrationId = integrationId; this.pendingMap = pendingMessages; this.processingTimeoutLatch = new CountDownLatch(pendingMap.size()); @@ -47,7 +46,7 @@ public boolean await(long packProcessingTimeout, TimeUnit timeUnit) throws Inter } public void onSuccess(UUID id) { - PublishIntegrationMsgProto msg = pendingMap.remove(id); + T msg = pendingMap.remove(id); if (msg != null) { processingTimeoutLatch.countDown(); } else { @@ -56,7 +55,7 @@ public void onSuccess(UUID id) { } public void onFailure(UUID id) { - PublishIntegrationMsgProto msg = pendingMap.remove(id); + T msg = pendingMap.remove(id); if (msg != null) { failedMap.put(id, msg); processingTimeoutLatch.countDown(); diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingResult.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingResult.java index c381507cd..fb91f6f18 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingResult.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/data/IntegrationPackProcessingResult.java @@ -16,20 +16,19 @@ package org.thingsboard.mqtt.broker.integration.api.data; import lombok.Getter; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import java.util.HashMap; import java.util.Map; import java.util.UUID; -public class IntegrationPackProcessingResult { +public class IntegrationPackProcessingResult { @Getter - private final Map pendingMap; + private final Map pendingMap; @Getter - private final Map failedMap; + private final Map failedMap; - public IntegrationPackProcessingResult(IntegrationPackProcessingContext ctx) { + public IntegrationPackProcessingResult(IntegrationPackProcessingContext ctx) { this.pendingMap = new HashMap<>(ctx.getPendingMap()); this.failedMap = new HashMap<>(ctx.getFailedMap()); } diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStats.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStats.java index 4bb530ab1..3793a3937 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStats.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStats.java @@ -25,7 +25,7 @@ public interface IntegrationProcessorStats { UUID getIntegrationUuid(); - void log(int totalMsgCount, IntegrationPackProcessingResult packProcessingResult, boolean finalIterationForPack); + void log(int totalMsgCount, IntegrationPackProcessingResult packProcessingResult, boolean finalIterationForPack); List getStatsCounters(); diff --git a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStatsImpl.java b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStatsImpl.java index 93c73b4aa..3fa2641ee 100644 --- a/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStatsImpl.java +++ b/common/integration/integration-api/src/main/java/org/thingsboard/mqtt/broker/integration/api/stats/IntegrationProcessorStatsImpl.java @@ -55,10 +55,10 @@ public class IntegrationProcessorStatsImpl implements IntegrationProcessorStats private final StatsCounter successIterationsCounter; private final StatsCounter failedIterationsCounter; - public IntegrationProcessorStatsImpl(UUID integrationUuid, StatsFactory statsFactory) { + public IntegrationProcessorStatsImpl(UUID integrationUuid, StatsFactory statsFactory, StatsType statsType) { this.integrationUuid = integrationUuid; String integrationId = integrationUuid.toString(); - String statsKey = StatsType.INTEGRATION_PROCESSOR.getPrintName(); + String statsKey = statsType.getPrintName(); this.totalMsgCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS, INTEGRATION_ID_TAG, integrationId); this.successMsgCounter = statsFactory.createStatsCounter(statsKey, SUCCESSFUL_MSGS, INTEGRATION_ID_TAG, integrationId); this.tmpTimeoutMsgCounter = statsFactory.createStatsCounter(statsKey, TMP_TIMEOUT, INTEGRATION_ID_TAG, integrationId); @@ -73,7 +73,7 @@ public IntegrationProcessorStatsImpl(UUID integrationUuid, StatsFactory statsFac } @Override - public void log(int totalMsgCount, IntegrationPackProcessingResult result, boolean finalIterationForPack) { + public void log(int totalMsgCount, IntegrationPackProcessingResult result, boolean finalIterationForPack) { int pending = result.getPendingMap().size(); int failed = result.getFailedMap().size(); int success = totalMsgCount - (pending + failed); diff --git a/common/integration/integration-api/src/test/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegrationLifecycleBodyTest.java b/common/integration/integration-api/src/test/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegrationLifecycleBodyTest.java new file mode 100644 index 000000000..c45e347f2 --- /dev/null +++ b/common/integration/integration-api/src/test/java/org/thingsboard/mqtt/broker/integration/api/AbstractIntegrationLifecycleBodyTest.java @@ -0,0 +1,310 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.api; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.integration.api.callback.IntegrationMsgCallback; +import org.thingsboard.mqtt.broker.integration.api.data.ContentType; +import org.thingsboard.mqtt.broker.integration.api.data.UplinkMetaData; +import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; +import org.thingsboard.mqtt.broker.gen.queue.RetainHandling; +import org.thingsboard.mqtt.broker.gen.queue.SubscriptionOptionsProto; +import org.thingsboard.mqtt.broker.gen.queue.TopicSubscriptionProto; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AbstractIntegrationLifecycleBodyTest { + + /** + * Minimal concrete subclass — only implements the one method the compiler requires + * (process, which AbstractIntegration leaves unimplemented). metadataTemplate is set + * directly before each test so constructLifecycleEventBody doesn't NPE. + */ + static class TestIntegration extends AbstractIntegration { + ObjectNode capturedLifecycleBody; + + @Override + public void process(PublishIntegrationMsgProto msg, IntegrationMsgCallback callback) { + // no-op — not exercised by these tests + } + + @Override + protected void doProcessLifecycleEvent(ObjectNode body, IntegrationMsgCallback callback) { + this.capturedLifecycleBody = body; + callback.onSuccess(); + } + + ObjectNode body(ClientLifecycleEventMsgProto msg) { + return constructLifecycleEventBody(msg); + } + } + + private TestIntegration integration; + + @BeforeEach + void setUp() { + integration = new TestIntegration(); + // Set an empty metadata template so constructLifecycleEventBody doesn't NPE. + integration.metadataTemplate = new UplinkMetaData(ContentType.JSON, Map.of()); + } + + // ── COMMON: username present in every event type ────────────────────────── + + @Test + void givenConnectedProto_whenBuildBody_thenUsernamePresent() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED") + .setClientId("c1") + .setUsername("alice") + .build(); + ObjectNode body = integration.body(msg); + assertEquals("alice", body.get("username").asText()); + } + + @Test + void givenNoUsernameOrIp_whenBuildBody_thenEmptyStringKeysOmitted() { + // auth disabled => no username; address-less session => no ipAddress + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED") + .setClientId("c1") + .build(); + ObjectNode body = integration.body(msg); + assertFalse(body.has("username")); + assertFalse(body.has("ipAddress")); + // no X.509 auth => no clientCertCn + assertFalse(body.has("clientCertCn")); + // the discriminator and populated fields stay present + assertEquals("CLIENT_CONNECTED", body.get("eventType").asText()); + assertEquals("c1", body.get("clientId").asText()); + } + + // ── COMMON: clientCertCn present for X.509-authenticated clients ────────── + + @Test + void givenX509Client_whenBuildBody_thenHasClientCertCn() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED") + .setClientId("c1") + .setClientCertCn("device-42") + .build(); + ObjectNode body = integration.body(msg); + assertEquals("device-42", body.get("clientCertCn").asText()); + } + + // ── CLIENT_CONNECTED: protocolVersion + sessionExpiryInterval ──────────── + + @Test + void givenConnectedProto_whenBuildBody_thenHasProtocolVersionAndSessionExpiryInterval() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED") + .setClientId("c1") + .setUsername("alice") + .setProtocolVersion(5) + .setSessionExpiryInterval(3600L) + .setCleanStart(true) + .setKeepAlive(60) + .build(); + ObjectNode body = integration.body(msg); + assertEquals(5, body.get("protocolVersion").asInt()); + assertEquals(3600L, body.get("sessionExpiryInterval").asLong()); + // existing fields must still be present + assertEquals(true, body.get("cleanStart").asBoolean()); + assertEquals(60, body.get("keepAlive").asInt()); + } + + // ── CLIENT_AUTHENTICATION_FAILED ────────────────────────────────────────── + + @Test + void givenAuthenticationFailedProto_whenBuildBody_thenHasReason() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_AUTHENTICATION_FAILED") + .setClientId("c2") + .setUsername("baduser") + .setReason("Bad credentials") + .build(); + ObjectNode body = integration.body(msg); + assertEquals("baduser", body.get("username").asText()); + assertEquals("Bad credentials", body.get("reason").asText()); + } + + @Test + void givenAuthenticationFailedProto_whenBuildBody_thenHasProtocolVersion() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_AUTHENTICATION_FAILED") + .setClientId("c2") + .setProtocolVersion(4) + .setReason("Bad credentials") + .build(); + ObjectNode body = integration.body(msg); + assertEquals(4, body.get("protocolVersion").asInt()); + } + + // ── CLIENT_AUTHORIZATION_FAILED ─────────────────────────────────────────── + + @Test + void givenAuthorizationFailedProto_whenBuildBody_thenHasActionTopicAndUsername() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_AUTHORIZATION_FAILED") + .setClientId("c1") + .setUsername("demo") + .setAction("publish") + .setTopic("zxc/demo/topic") + .build(); + ObjectNode body = integration.body(msg); + assertEquals("demo", body.get("username").asText()); + assertEquals("publish", body.get("action").asText()); + assertEquals("zxc/demo/topic", body.get("topic").asText()); + } + + // ── CLIENT_CONNECTION_FAILED ────────────────────────────────────────────── + + @Test + void givenConnectionFailedProto_whenBuildBody_thenHasReason() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTION_FAILED") + .setClientId("c3") + .setReason("QUOTA_EXCEEDED") + .build(); + ObjectNode body = integration.body(msg); + assertEquals("QUOTA_EXCEEDED", body.get("reason").asText()); + } + + // ── CLIENT_SUBSCRIBED: full subscription details ────────────────────────── + + @Test + void givenSubscribedProto_whenBuildBody_thenHasFullSubscriptionDetails() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_SUBSCRIBED") + .setClientId("c1") + .addSubscriptions(TopicSubscriptionProto.newBuilder() + .setTopic("foo/bar") + .setQos(1) + .setShareName("g1") + .setSubscriptionId(7) + .setOptions(SubscriptionOptionsProto.newBuilder() + .setNoLocal(true) + .setRetainAsPublish(true) + .setRetainHandling(RetainHandling.DONT_SEND) + .build()) + .build()) + .build(); + ObjectNode body = integration.body(msg); + JsonNode sub = body.get("subscriptions").get(0); + assertEquals("foo/bar", sub.get("topicFilter").asText()); + assertEquals(1, sub.get("qos").asInt()); + assertEquals("g1", sub.get("shareName").asText()); + assertEquals(7, sub.get("subscriptionId").asInt()); + assertTrue(sub.get("options").get("noLocal").asBoolean()); + assertTrue(sub.get("options").get("retainAsPublish").asBoolean()); + assertEquals("DONT_SEND", sub.get("options").get("retainHandling").asText()); + } + + @Test + void givenSubscribedProtoWithoutShareOrSubId_whenBuildBody_thenOptionalKeysOmitted() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_SUBSCRIBED") + .setClientId("c1") + .addSubscriptions(TopicSubscriptionProto.newBuilder() + .setTopic("foo/bar") + .setQos(0) + .setOptions(SubscriptionOptionsProto.getDefaultInstance()) + .build()) + .build(); + ObjectNode body = integration.body(msg); + JsonNode sub = body.get("subscriptions").get(0); + assertFalse(sub.has("shareName")); + assertFalse(sub.has("subscriptionId")); + assertEquals("SEND", sub.get("options").get("retainHandling").asText()); + } + + @Test + void givenUnsubscribedProto_whenBuildBody_thenHasTopicFilterAndShareName() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_UNSUBSCRIBED") + .setClientId("c1") + .addSubscriptions(TopicSubscriptionProto.newBuilder().setTopic("foo/bar").build()) + .addSubscriptions(TopicSubscriptionProto.newBuilder().setTopic("baz").setShareName("g1").build()) + .build(); + ObjectNode body = integration.body(msg); + JsonNode subs = body.get("subscriptions"); + assertEquals(2, subs.size()); + // regular unsubscribe: bare topic filter, no shareName + assertEquals("foo/bar", subs.get(0).get("topicFilter").asText()); + assertFalse(subs.get(0).has("shareName")); + // shared unsubscribe: bare filter + shareName so $share/g1/baz can be reconstructed + assertEquals("baz", subs.get(1).get("topicFilter").asText()); + assertEquals("g1", subs.get(1).get("shareName").asText()); + // an UNSUBSCRIBE names filters only — no qos/options/subscriptionId + assertFalse(subs.get(0).has("qos")); + assertFalse(subs.get(0).has("options")); + assertFalse(subs.get(0).has("subscriptionId")); + } + + // ── processLifecycleEvent delivers the ObjectNode (not a serialized String) ── + + @Test + void givenProto_whenProcessLifecycleEvent_thenDoProcessReceivesConstructedObjectNode() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED") + .setClientId("c1") + .setUsername("alice") + .build(); + + AtomicBoolean succeeded = new AtomicBoolean(false); + IntegrationMsgCallback callback = new IntegrationMsgCallback() { + @Override + public void onSuccess() { + succeeded.set(true); + } + + @Override + public void onFailure(Throwable t) { + } + }; + + integration.processLifecycleEvent(msg, callback); + + // doProcessLifecycleEvent receives the parsed ObjectNode (equal to constructLifecycleEventBody's output), + // so JSON-native transports like HTTP can forward it as application/json rather than a text/plain String. + assertNotNull(integration.capturedLifecycleBody); + assertEquals(integration.body(msg), integration.capturedLifecycleBody); + assertEquals("alice", integration.capturedLifecycleBody.get("username").asText()); + assertTrue(succeeded.get()); + } + + // ── metadata node always present ────────────────────────────────────────── + + @Test + void givenAnyProto_whenBuildBody_thenMetadataNodePresent() { + ClientLifecycleEventMsgProto msg = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_DISCONNECTED") + .setClientId("c1") + .setUsername("u") + .build(); + ObjectNode body = integration.body(msg); + assertNotNull(body.get("metadata")); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/kafka/settings/integration/IntegrationEventKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/kafka/settings/integration/IntegrationEventKafkaSettings.java new file mode 100644 index 000000000..a0e3439ee --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/kafka/settings/integration/IntegrationEventKafkaSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.queue.kafka.settings.integration; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.thingsboard.mqtt.broker.queue.kafka.settings.AbstractKafkaSettings; + +@Data +@Component +@ConfigurationProperties(prefix = "queue.kafka.integration-event") +public class IntegrationEventKafkaSettings extends AbstractKafkaSettings { + + private String topicProperties; + private String additionalProducerConfig; + private String additionalConsumerConfig; + +} diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/ExecutorIntegrationMsgQueueProvider.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/ExecutorIntegrationMsgQueueProvider.java index 3688e0da6..b0315ee31 100644 --- a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/ExecutorIntegrationMsgQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/ExecutorIntegrationMsgQueueProvider.java @@ -18,6 +18,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; import org.thingsboard.mqtt.broker.queue.TbQueueProducer; @@ -54,6 +55,21 @@ public Map getTopicConfigs() { return integrationMsgQueueFactory.getTopicConfigs(); } + @Override + public TbQueueProducer> getIeEventMsgProducer() { + throw new RuntimeException(TBMQ_IE_NOT_IMPLEMENTED); + } + + @Override + public TbQueueControlledOffsetConsumer> getIeEventMsgConsumer(String topic, String consumerGroupId, String integrationId) { + return integrationMsgQueueFactory.createEventConsumer(topic, consumerGroupId, getConsumerId(integrationId)); + } + + @Override + public Map getIeEventMsgTopicConfigs() { + return integrationMsgQueueFactory.getEventTopicConfigs(); + } + private String getConsumerId(String integrationId) { return serviceInfoProvider.getServiceId() + HYPHEN + integrationId; } diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueFactory.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueFactory.java index 4285d6def..6737c07ab 100644 --- a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.mqtt.broker.queue.provider.integration; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; import org.thingsboard.mqtt.broker.queue.TbQueueProducer; @@ -29,4 +30,10 @@ public interface IntegrationMsgQueueFactory { TbQueueControlledOffsetConsumer> createConsumer(String topic, String consumerGroupId, String consumerId); Map getTopicConfigs(); + + TbQueueProducer> createEventProducer(String serviceId); + + TbQueueControlledOffsetConsumer> createEventConsumer(String topic, String consumerGroupId, String consumerId); + + Map getEventTopicConfigs(); } diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueProvider.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueProvider.java index 39d6e78d6..ae496090c 100644 --- a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/IntegrationMsgQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.mqtt.broker.queue.provider.integration; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; import org.thingsboard.mqtt.broker.queue.TbQueueProducer; @@ -30,4 +31,10 @@ public interface IntegrationMsgQueueProvider { Map getTopicConfigs(); + TbQueueProducer> getIeEventMsgProducer(); + + TbQueueControlledOffsetConsumer> getIeEventMsgConsumer(String topic, String consumerGroupId, String integrationId); + + Map getIeEventMsgTopicConfigs(); + } diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/KafkaIntegrationMsgQueueFactory.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/KafkaIntegrationMsgQueueFactory.java index db3163d59..8474dd622 100644 --- a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/KafkaIntegrationMsgQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/KafkaIntegrationMsgQueueFactory.java @@ -19,12 +19,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; import org.thingsboard.mqtt.broker.queue.TbQueueProducer; import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; import org.thingsboard.mqtt.broker.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.mqtt.broker.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.mqtt.broker.queue.kafka.settings.integration.IntegrationEventKafkaSettings; import org.thingsboard.mqtt.broker.queue.kafka.settings.integration.IntegrationMsgKafkaSettings; import org.thingsboard.mqtt.broker.queue.provider.AbstractQueueFactory; import org.thingsboard.mqtt.broker.queue.util.QueueUtil; @@ -38,12 +40,15 @@ public class KafkaIntegrationMsgQueueFactory extends AbstractQueueFactory implements IntegrationMsgQueueFactory { private final IntegrationMsgKafkaSettings integrationMsgKafkaSettings; + private final IntegrationEventKafkaSettings integrationEventKafkaSettings; private Map topicConfigs; + private Map eventTopicConfigs; @PostConstruct public void init() { this.topicConfigs = validateAndConfigurePartitionsForTopic(integrationMsgKafkaSettings.getTopicProperties(), "IE message"); + this.eventTopicConfigs = validateAndConfigurePartitionsForTopic(integrationEventKafkaSettings.getTopicProperties(), "IE event message"); } @Override @@ -81,4 +86,43 @@ public TbQueueControlledOffsetConsumer getTopicConfigs() { return topicConfigs; } + + @Override + public TbQueueProducer> createEventProducer(String serviceId) { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> producerBuilder = TbKafkaProducerTemplate.builder(); + producerBuilder.properties(producerSettings.toProps(integrationEventKafkaSettings.getAdditionalProducerConfig())); + producerBuilder.clientId(kafkaPrefix + "ie-event-msg-producer-" + serviceId); + producerBuilder.admin(queueAdmin); + producerBuilder.topicConfigs(eventTopicConfigs); + producerBuilder.statsManager(producerStatsManager); + // IE owns events-topic provisioning (IntegrationTopicServiceImpl.createEventTopic); best-effort + // production on the MQTT hot path must not block on a synchronous admin topic-creation call. + producerBuilder.createTopicIfNotExists(false); + return producerBuilder.build(); + } + + @Override + public TbQueueControlledOffsetConsumer> createEventConsumer(String topic, String consumerGroupId, String consumerId) { + String clientId = "ie-event-msg-consumer-" + consumerId; + + Properties props = consumerSettings.toProps(topic, integrationEventKafkaSettings.getAdditionalConsumerConfig()); + QueueUtil.overrideProperties("IeEventMsgQueue-" + consumerId, props, requiredConsumerProperties); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.properties(props); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ClientLifecycleEventMsgProto.parseFrom(msg.getData()), msg.getHeaders(), + msg.getPartition(), msg.getOffset())); + consumerBuilder.clientId(kafkaPrefix + clientId); + consumerBuilder.groupId(consumerGroupId); + consumerBuilder.topic(topic); + consumerBuilder.statsService(consumerStatsService); + consumerBuilder.autoCommit(false); + consumerBuilder.createTopicIfNotExists(false); + return consumerBuilder.build(); + } + + @Override + public Map getEventTopicConfigs() { + return eventTopicConfigs; + } } diff --git a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/TbmqIntegrationMsgQueueProvider.java b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/TbmqIntegrationMsgQueueProvider.java index b582f4fd8..8ebd42416 100644 --- a/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/TbmqIntegrationMsgQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/mqtt/broker/queue/provider/integration/TbmqIntegrationMsgQueueProvider.java @@ -20,6 +20,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; import org.thingsboard.mqtt.broker.queue.TbQueueProducer; @@ -41,10 +42,12 @@ public class TbmqIntegrationMsgQueueProvider implements IntegrationMsgQueueProvi private final ServiceInfoProvider serviceInfoProvider; private TbQueueProducer> integrationMsgProducer; + private TbQueueProducer> integrationEventMsgProducer; @PostConstruct public void init() { this.integrationMsgProducer = integrationMsgQueueFactory.createProducer(serviceInfoProvider.getServiceId()); + this.integrationEventMsgProducer = integrationMsgQueueFactory.createEventProducer(serviceInfoProvider.getServiceId()); } @PreDestroy @@ -52,6 +55,9 @@ public void destroy() { if (integrationMsgProducer != null) { integrationMsgProducer.stop(); } + if (integrationEventMsgProducer != null) { + integrationEventMsgProducer.stop(); + } } @Override @@ -68,4 +74,19 @@ public TbQueueControlledOffsetConsumer getTopicConfigs() { throw new RuntimeException(TBMQ_NOT_IMPLEMENTED); } + + @Override + public TbQueueProducer> getIeEventMsgProducer() { + return integrationEventMsgProducer; + } + + @Override + public TbQueueControlledOffsetConsumer> getIeEventMsgConsumer(String topic, String consumerGroupId, String integrationId) { + throw new RuntimeException(TBMQ_NOT_IMPLEMENTED); + } + + @Override + public Map getIeEventMsgTopicConfigs() { + throw new RuntimeException(TBMQ_NOT_IMPLEMENTED); + } } diff --git a/common/queue/src/main/proto/integration.proto b/common/queue/src/main/proto/integration.proto index 41539e465..4e95ffe33 100644 --- a/common/queue/src/main/proto/integration.proto +++ b/common/queue/src/main/proto/integration.proto @@ -103,3 +103,23 @@ message PublishIntegrationMsgProto { string tbmqNode = 2; int64 timestamp = 3; } + +message ClientLifecycleEventMsgProto { + string eventType = 1; + string clientId = 2; + string username = 3; + string sessionId = 4; + string ipAddress = 5; + int64 ts = 6; + string tbmqNode = 7; + bool cleanStart = 8; + int32 keepAlive = 9; + int32 protocolVersion = 10; + string disconnectReason = 11; + repeated queue.TopicSubscriptionProto subscriptions = 12; // CLIENT_SUBSCRIBED: granted subscriptions (full); CLIENT_UNSUBSCRIBED: removed subscriptions (topic + shareName only) + string reason = 13; // CLIENT_AUTHENTICATION_FAILED / CLIENT_CONNECTION_FAILED: failure reason + string action = 14; // CLIENT_AUTHORIZATION_FAILED: publish|subscribe + string topic = 15; // CLIENT_AUTHORIZATION_FAILED: denied topic/filter + int64 sessionExpiryInterval = 16; // CLIENT_CONNECTED enrichment + string clientCertCn = 17; // X.509 certificate Common Name; common field present only for X.509-authenticated clients +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 2b0347934..5856a6302 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -179,10 +179,17 @@ message ClientSessionStatsCleanupProto { string clientId = 1; } +message IntegrationLifecycleConfigProto { + string integrationId = 1; + bool deleted = 2; + repeated string lifecycleEventTypes = 3; +} + message InternodeNotificationProto { MqttAuthSettingsProto mqttAuthSettingsProto = 1; MqttAuthProviderProto mqttAuthProviderProto = 2; ClientSessionStatsCleanupProto clientSessionStatsCleanupProto = 3; + IntegrationLifecycleConfigProto integrationLifecycleConfigProto = 4; } message DevicePublishMsgProto { diff --git a/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsConstantNames.java b/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsConstantNames.java index e6fffe836..0dbdc3acb 100644 --- a/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsConstantNames.java +++ b/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsConstantNames.java @@ -53,4 +53,6 @@ public class StatsConstantNames { public static final String MSG_TYPE = "msgType"; + public static final String DROPPED_LIFECYCLE_EVENTS = "droppedLifecycleEvents"; + } diff --git a/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsType.java index 6664be0e6..8c3c07ca8 100644 --- a/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/mqtt/broker/common/stats/StatsType.java @@ -60,6 +60,7 @@ public enum StatsType { IE_UPLINK_PRODUCER("ie.uplink.published"), INTEGRATION("integration"), INTEGRATION_PROCESSOR("integrationProcessor"), + INTEGRATION_EVENT_PROCESSOR("integrationEventProcessor"), ; private final String printName; diff --git a/dao/src/main/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImpl.java b/dao/src/main/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImpl.java index 8e3cbc531..de99f3f8d 100644 --- a/dao/src/main/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImpl.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; import org.thingsboard.mqtt.broker.common.data.integration.Integration; import org.thingsboard.mqtt.broker.common.data.page.PageData; import org.thingsboard.mqtt.broker.common.data.page.PageLink; @@ -135,14 +136,32 @@ protected void validateDataImpl(Integration integration) { if (integration.getType() == null) { throw new DataValidationException("Integration type should be specified!"); } - JsonNode topicFilters = integration.getConfiguration().get("topicFilters"); - if (topicFilters == null || topicFilters.isNull()) { - throw new DataValidationException("Topic filters should be specified!"); + JsonNode configuration = integration.getConfiguration(); + JsonNode topicFilters = configuration.get("topicFilters"); + JsonNode lifecycleEventTypes = configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY); + + boolean hasTopicFilters = topicFilters != null && topicFilters.isArray() && !topicFilters.isEmpty(); + boolean hasLifecycleEventTypes = lifecycleEventTypes != null && lifecycleEventTypes.isArray() && !lifecycleEventTypes.isEmpty(); + + if (!hasTopicFilters && !hasLifecycleEventTypes) { + throw new DataValidationException("Either topic filters or lifecycle event types should be specified!"); } - if (!topicFilters.isArray()) { - throw new DataValidationException("Topic filters should be an array!"); + + if (topicFilters != null && !topicFilters.isNull()) { + if (!topicFilters.isArray()) { + throw new DataValidationException("Topic filters should be an array!"); + } + topicFilters.forEach(topicFilter -> topicValidationService.validateTopicFilter(topicFilter.asText())); + } + + if (lifecycleEventTypes != null && !lifecycleEventTypes.isNull()) { + if (!lifecycleEventTypes.isArray()) { + throw new DataValidationException("Lifecycle event types should be an array!"); + } + ClientLifecycleEventTypeUtil.parse(lifecycleEventTypes, name -> { + throw new DataValidationException("Unknown lifecycle event type: " + name); + }); } - topicFilters.forEach(topicFilter -> topicValidationService.validateTopicFilter(topicFilter.asText())); } }; diff --git a/dao/src/test/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImplTest.java b/dao/src/test/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImplTest.java index c1299ff87..3e23609a7 100644 --- a/dao/src/test/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImplTest.java +++ b/dao/src/test/java/org/thingsboard/mqtt/broker/dao/integration/IntegrationServiceImplTest.java @@ -113,6 +113,139 @@ public void testUpdateIntegrationType() { integrationService.deleteIntegration(savedIntegration); } + // Mode 3: events-only integration is now valid (no topicFilters). + @Test + public void testSaveEventsOnlyIntegration() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("lifecycleEventTypes").add("CLIENT_CONNECTED").add("CLIENT_DISCONNECTED"); + + Integration integration = new Integration(); + integration.setName("Events only integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + Integration saved = integrationService.saveIntegration(integration); + Assert.assertNotNull(saved); + Assert.assertNotNull(saved.getId()); + savedIntegrations.add(saved); + } + + // Mode 1: messages + events integration is valid. + @Test + public void testSaveMessagesAndEventsIntegration() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("topicFilters").add("tbmq/#"); + configuration.putArray("lifecycleEventTypes").add("CLIENT_CONNECTED"); + + Integration integration = new Integration(); + integration.setName("Messages and events integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + Integration saved = integrationService.saveIntegration(integration); + Assert.assertNotNull(saved); + savedIntegrations.add(saved); + } + + // Mode 2 (regression guard): legacy messages-only integration still valid. + @Test + public void testSaveLegacyMessagesOnlyIntegration() { + Integration integration = new Integration(); + integration.setName("Messages only integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(getIntegrationConfiguration()); // topicFilters: ["#"], no lifecycleEventTypes + + Integration saved = integrationService.saveIntegration(integration); + Assert.assertNotNull(saved); + savedIntegrations.add(saved); + } + + // Neither list -> rejected with the new combined message. + @Test + public void testSaveIntegrationWithNoTopicFiltersAndNoEventsIsRejected() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + + Integration integration = new Integration(); + integration.setName("No filters no events integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + DataValidationException ex = Assertions.assertThrows(DataValidationException.class, + () -> integrationService.saveIntegration(integration)); + Assertions.assertTrue(ex.getMessage().contains("lifecycle event types")); + } + + // Empty topicFilters array with no events -> rejected (old code accepted an empty array). + @Test + public void testSaveIntegrationWithEmptyTopicFiltersAndNoEventsIsRejected() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("topicFilters"); // empty array + + Integration integration = new Integration(); + integration.setName("Empty filters no events integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + DataValidationException ex = Assertions.assertThrows(DataValidationException.class, + () -> integrationService.saveIntegration(integration)); + Assertions.assertTrue(ex.getMessage().contains("lifecycle event types")); + } + + // Unknown lifecycle event-type name -> rejected. + @Test + public void testSaveIntegrationWithUnknownLifecycleEventTypeIsRejected() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("lifecycleEventTypes").add("CLIENT_CONNECTED").add("NOT_A_REAL_EVENT"); + + Integration integration = new Integration(); + integration.setName("Unknown event type integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + DataValidationException ex = Assertions.assertThrows(DataValidationException.class, + () -> integrationService.saveIntegration(integration)); + Assertions.assertTrue(ex.getMessage().contains("Unknown lifecycle event type")); + } + + // Non-array lifecycleEventTypes -> rejected. + @Test + public void testSaveIntegrationWithNonArrayLifecycleEventTypesIsRejected() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("topicFilters").add("#"); + configuration.put("lifecycleEventTypes", "CLIENT_CONNECTED"); // not an array + + Integration integration = new Integration(); + integration.setName("Non-array events integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + DataValidationException ex = Assertions.assertThrows(DataValidationException.class, + () -> integrationService.saveIntegration(integration)); + Assertions.assertTrue(ex.getMessage().contains("should be an array")); + } + + // Existing behavior preserved: an invalid topic filter is still rejected. + @Test + public void testSaveIntegrationWithInvalidTopicFilterIsRejected() { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.putObject("metadata").put("key1", "val1"); + configuration.putArray("topicFilters").add("a/#/b"); // multi-level wildcard not last -> invalid + + Integration integration = new Integration(); + integration.setName("Invalid topic filter integration"); + integration.setType(IntegrationType.HTTP); + integration.setConfiguration(configuration); + + Assertions.assertThrows(DataValidationException.class, + () -> integrationService.saveIntegration(integration)); + } + @Test public void testFindIntegrationById() { Integration integration = new Integration(); diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/AbstractHttpIntegration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/AbstractHttpIntegration.java index 0550186a3..95483d1ad 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/AbstractHttpIntegration.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/AbstractHttpIntegration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.mqtt.broker.integration.service.integration.http; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.mqtt.broker.common.data.BasicCallback; import org.thingsboard.mqtt.broker.common.data.util.CallbackUtil; @@ -36,6 +37,17 @@ public void process(PublishIntegrationMsgProto msg, IntegrationMsgCallback integ } } + @Override + protected void doProcessLifecycleEvent(ObjectNode body, IntegrationMsgCallback integrationMsgCallback) { + var callback = createBasicCallback(integrationMsgCallback); + try { + doSendBody(body, callback); + } catch (Exception e) { + log.error("[{}][{}] Failure during lifecycle event processing", lifecycleMsg.getIntegrationId(), lifecycleMsg.getName(), e); + callback.onFailure(e); + } + } + private BasicCallback createBasicCallback(IntegrationMsgCallback callback) { return CallbackUtil.createCallback( () -> { @@ -49,4 +61,9 @@ private BasicCallback createBasicCallback(IntegrationMsgCallback callback) { } protected abstract void doProcess(PublishIntegrationMsgProto msg, BasicCallback callback) throws Exception; + + protected void doSendBody(ObjectNode body, BasicCallback callback) throws Exception { + log.debug("[{}][{}] doSendBody is not implemented for this HTTP integration", lifecycleMsg.getIntegrationId(), lifecycleMsg.getName()); + callback.onSuccess(); + } } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/HttpIntegration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/HttpIntegration.java index c4a2cb477..db6fbc1a1 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/HttpIntegration.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/http/HttpIntegration.java @@ -75,6 +75,11 @@ protected void doProcess(PublishIntegrationMsgProto msg, BasicCallback callback) tbHttpClient.processMessage(getRequestBody(msg), callback); } + @Override + protected void doSendBody(ObjectNode body, BasicCallback callback) { + tbHttpClient.processMessage(body, callback); + } + private Object getRequestBody(PublishIntegrationMsgProto msg) { ByteString payload = msg.getPublishMsgProto().getPayload(); if (config.isSendOnlyMsgPayload()) { diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/kafka/KafkaIntegration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/kafka/KafkaIntegration.java index db461e9fd..77fee7e7f 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/kafka/KafkaIntegration.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/kafka/KafkaIntegration.java @@ -16,6 +16,7 @@ package org.thingsboard.mqtt.broker.integration.service.integration.kafka; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.Admin; @@ -32,6 +33,7 @@ import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardErrorCode; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; import org.thingsboard.mqtt.broker.common.data.integration.Integration; +import org.thingsboard.mqtt.broker.common.util.JacksonUtil; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.integration.api.AbstractIntegration; import org.thingsboard.mqtt.broker.integration.api.IntegrationContext; @@ -169,6 +171,40 @@ private void publish(PublishIntegrationMsgProto msg, IntegrationMsgCallback call } } + @Override + protected void doProcessLifecycleEvent(ObjectNode body, IntegrationMsgCallback callback) { + String value = JacksonUtil.toString(body); + context.getExternalCallExecutor().executeAsync(() -> { + publishBody(value, callback); + return null; + }); + } + + private void publishBody(String body, IntegrationMsgCallback callback) { + try { + Headers headers = new RecordHeaders(); + config.getKafkaHeaders().forEach((k, v) -> headers.add(new RecordHeader(k, v.getBytes(config.getKafkaHeadersCharset())))); + + var kvProducerRecord = new ProducerRecord<>(config.getTopic(), null, config.getKey(), body, headers); + producer.send(kvProducerRecord, (metadata, e) -> { + if (e == null) { + log.debug("[{}][{}] publishBody success {}{}{}", getId(), getName(), metadata.topic(), + metadata.partition(), metadata.offset()); + integrationStatistics.incMessagesProcessed(); + callback.onSuccess(); + } else { + log.warn("[{}][{}] processException", getId(), getName(), e); + handleMsgProcessingFailure(e); + callback.onFailure(e); + } + }); + } catch (Exception e) { + log.warn("[{}][{}] Failed to publish lifecycle event body", getId(), getName(), e); + handleMsgProcessingFailure(e); + callback.onFailure(e); + } + } + private String getRecordValue(PublishIntegrationMsgProto msg) { if (config.isSendOnlyMsgPayload()) { ByteString payload = msg.getPublishMsgProto().getPayload(); diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidator.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidator.java index 2495dc7c1..77940bc35 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidator.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidator.java @@ -63,6 +63,22 @@ private static void validateTopicName(MqttIntegrationConfig mqttIntegrationConfi if (mqttIntegrationConfig.isUseMsgTopicName()) { return; } + validateTopicShape(topicName); + } + + /** + * Lifecycle events have no originating MQTT message to derive a topic from, so they are always published to a + * dedicated static events topic. It must be present and a valid publish topic. + */ + public static void validateEventsTopic(MqttIntegrationConfig mqttIntegrationConfig) { + var eventsTopicName = mqttIntegrationConfig.getEventsTopicName(); + if (StringUtils.isEmpty(eventsTopicName)) { + throw new IllegalArgumentException("Topic name is required to deliver lifecycle events"); + } + validateTopicShape(eventsTopicName); + } + + private static void validateTopicShape(String topicName) { if (topicName.contains(MULTI_LEVEL_WILDCARD) || topicName.contains(SINGLE_LEVEL_WILDCARD)) { throw new IllegalArgumentException("Topic name cannot contain wildcard characters"); diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegration.java index fd100c1e2..798730d5c 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegration.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegration.java @@ -16,6 +16,7 @@ package org.thingsboard.mqtt.broker.integration.service.integration.mqtt; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; @@ -29,6 +30,7 @@ import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardErrorCode; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; import org.thingsboard.mqtt.broker.common.data.integration.Integration; +import org.thingsboard.mqtt.broker.common.util.JacksonUtil; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.integration.api.AbstractIntegration; import org.thingsboard.mqtt.broker.integration.api.IntegrationContext; @@ -67,6 +69,17 @@ public void doValidateConfiguration(JsonNode clientConfiguration, boolean allowL } } + @Override + protected void doValidateLifecycleEventsDelivery(JsonNode clientConfiguration) throws ThingsboardException { + try { + // Lifecycle events have no incoming MQTT message, so they are always published to the dedicated static + // events topic (config.getEventsTopicName()) - require it to be set and a valid publish topic. + MqttConfigValidator.validateEventsTopic(getClientConfiguration(clientConfiguration, MqttIntegrationConfig.class)); + } catch (Exception e) { + throw new ThingsboardException(e.getMessage(), ThingsboardErrorCode.GENERAL); + } + } + @Override public void doCheckConnection(Integration integration, IntegrationContext ctx) throws ThingsboardException { this.context = ctx; @@ -132,6 +145,26 @@ public void process(PublishIntegrationMsgProto msg, IntegrationMsgCallback callb ); } + @Override + protected void doProcessLifecycleEvent(ObjectNode body, IntegrationMsgCallback callback) { + // Lifecycle events have no originating message, so they always go to the dedicated static events topic with + // fixed QoS 1 (at-least-once, do not silently drop events) and retain=false (shared stream across all clients). + client.publish(config.getEventsTopicName(), Unpooled.wrappedBuffer(JacksonUtil.toString(body).getBytes(StandardCharsets.UTF_8)), + MqttQoS.AT_LEAST_ONCE, false) + .addListener(future -> { + if (future.isSuccess()) { + log.debug("[{}][{}] lifecycle event publish success {}", getId(), getName(), config.getEventsTopicName()); + integrationStatistics.incMessagesProcessed(); + callback.onSuccess(); + } else { + var t = future.cause(); + log.warn("[{}][{}] lifecycle event processException", getId(), getName(), t); + handleMsgProcessingFailure(t); + callback.onFailure(t); + } + }); + } + private String getMsgTopicName(PublishIntegrationMsgProto msg) { return config.isUseMsgTopicName() ? msg.getPublishMsgProto().getTopicName() : config.getTopicName(); } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationConfig.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationConfig.java index 10042a9f3..b1478ff2b 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationConfig.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationConfig.java @@ -35,6 +35,7 @@ public class MqttIntegrationConfig { private int port; private String topicName; private boolean useMsgTopicName; + private String eventsTopicName; private String clientId; private ClientCredentials credentials; private boolean ssl; diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtil.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtil.java new file mode 100644 index 000000000..b13dbc7cf --- /dev/null +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtil.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.thingsboard.mqtt.broker.common.data.integration.ClientLifecycleEventTypeUtil; +import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class IntegrationEventsOptInUtil { + + public static boolean isOptedIn(IntegrationLifecycleMsg lifecycleMsg) { + if (lifecycleMsg == null) { + return false; + } + JsonNode configuration = lifecycleMsg.getConfiguration(); + return configuration != null + && configuration.has(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY) + && !configuration.get(ClientLifecycleEventTypeUtil.LIFECYCLE_EVENT_TYPES_KEY).isEmpty(); + } +} diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImpl.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImpl.java index 2104cf47a..786ca8908 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImpl.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImpl.java @@ -16,6 +16,7 @@ package org.thingsboard.mqtt.broker.integration.service.processing; import com.google.common.collect.Maps; +import com.google.protobuf.GeneratedMessageV3; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; @@ -24,6 +25,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.mqtt.broker.common.data.util.CallbackUtil; import org.thingsboard.mqtt.broker.common.util.ThingsBoardExecutors; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.integration.api.IntegrationStatisticsService; import org.thingsboard.mqtt.broker.integration.api.TbPlatformIntegration; @@ -53,6 +55,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import static java.lang.Long.MAX_VALUE; @@ -62,6 +65,7 @@ public class IntegrationMsgProcessorImpl implements IntegrationMsgProcessor { private final ConcurrentMap integrations = new ConcurrentHashMap<>(); + private final ConcurrentMap eventIntegrations = new ConcurrentHashMap<>(); private final IntegrationMsgQueueProvider integrationMsgQueueProvider; private final IntegrationTopicService integrationTopicService; @@ -77,6 +81,10 @@ public class IntegrationMsgProcessorImpl implements IntegrationMsgProcessor { private long pollDuration; @Value("${queue.integration-msg.pack-processing-timeout:30000}") private long packProcessingTimeout; + @Value("${queue.integration-event.poll-interval:100}") + private long eventPollDuration; + @Value("${queue.integration-event.pack-processing-timeout:30000}") + private long eventPackProcessingTimeout; @PostConstruct public void init() { @@ -91,6 +99,8 @@ public void destroy() { stopped = true; integrations.forEach((integrationId, integration) -> stopIntegrationCancelTask(integration)); integrations.clear(); + eventIntegrations.forEach((id, holder) -> stopIntegrationCancelTask(holder)); + eventIntegrations.clear(); if (integrationMsgsConsumerExecutor != null) { ThingsBoardExecutors.shutdownAndAwaitTermination(integrationMsgsConsumerExecutor, "IE msg consumers'"); } @@ -120,6 +130,11 @@ public void startProcessingIntegrationMessages(TbPlatformIntegration integration }); integrationHolder.setFuture(future); integrations.put(integrationId, integrationHolder); + try { + startProcessingEventsIfOptedIn(integration); + } catch (Exception e) { + log.warn("[{}] Failed to start lifecycle-events processing; continuing with data only", integrationId, e); + } } @Override @@ -136,6 +151,15 @@ public void stopProcessingIntegrationMessages(String integrationId) { log.warn("[{}] Exception stopping future for integration", integrationId, e); } } + IntegrationHolder eventHolder = eventIntegrations.remove(integrationId); + if (eventHolder != null) { + try { + stopIntegrationCancelTask(eventHolder); + statsService.ifPresent(svc -> svc.clearIntegrationEventProcessorStats(eventHolder.getIntegrationUuid())); + } catch (Exception e) { + log.warn("[{}] Exception stopping events future for integration", integrationId, e); + } + } } @Override @@ -147,6 +171,10 @@ public void clearIntegrationMessages(String integrationId) { () -> { }, throwable -> { })); + integrationTopicService.deleteEventTopic(integrationId, CallbackUtil.createCallback( + () -> { + }, throwable -> { + })); } catch (Exception e) { log.warn("[{}] Exception clearing consumer group and topic for integration", integrationId, e); } @@ -179,46 +207,133 @@ private TbQueueControlledOffsetConsumer> consumer, - IntegrationHolder integrationHolder) { - final AtomicLong counter = new AtomicLong(0); + private void startProcessingEventsIfOptedIn(TbPlatformIntegration integration) { + String integrationId = integration.getIntegrationId(); + if (!IntegrationEventsOptInUtil.isOptedIn(integration.getLifecycleMsg())) { + log.debug("[{}] Integration is not opted in for lifecycle events. Skipping events consumer start", integrationId); + return; + } + if (eventIntegrations.containsKey(integrationId)) { + log.info("[{}] The events processor is already running. Skipping start", integrationId); + return; + } + String eventTopic = integrationTopicService.createEventTopic(integrationId); + log.info("[{}] Starting integration lifecycle-events processing", integrationId); + TbQueueControlledOffsetConsumer> consumer = + initEventConsumer(integrationId, eventTopic); + IntegrationHolder holder = new IntegrationHolder(integration); + Future future = integrationMsgsConsumerExecutor.submit(() -> { + try { + processEvents(consumer, holder); + } finally { + consumer.unsubscribeAndClose(); + } + }); + holder.setFuture(future); + eventIntegrations.put(integrationId, holder); + } + + private TbQueueControlledOffsetConsumer> initEventConsumer(String integrationId, String topic) { + TbQueueControlledOffsetConsumer> consumer = + integrationMsgQueueProvider.getIeEventMsgConsumer( + topic, integrationTopicService.getEventConsumerGroup(integrationId), integrationId); + try { + consumer.assignPartition(0); + Optional committedOffset = consumer.getCommittedOffset(consumer.getTopic(), 0); + if (committedOffset.isEmpty()) { + long endOffset = consumer.getEndOffset(consumer.getTopic(), 0); + consumer.commit(0, endOffset); + } + return consumer; + } catch (Exception e) { + log.error("[{}] Failed to init integration events consumer", integrationId, e); + consumer.unsubscribeAndClose(); + throw e; + } + } + + private void processEvents(TbQueueControlledOffsetConsumer> consumer, + IntegrationHolder holder) { + IntegrationProcessorStats stats = statsService + .map(svc -> svc.createIntegrationEventProcessorStats(holder.getIntegrationUuid())) + .orElse(null); + processQueue(consumer, holder, this::dispatchEvent, stats, "events", + eventPollDuration, eventPackProcessingTimeout, + () -> ackStrategyFactory.newEventInstance(holder.getIntegrationId())); + } + void dispatchEvent(IntegrationHolder holder, UUID packetId, ClientLifecycleEventMsgProto event, + IntegrationPackProcessingContext ctx) { + holder.getIntegration().processLifecycleEvent(event, new BaseIntegrationMsgCallback(packetId, ctx)); + } + + private void processMessages(TbQueueControlledOffsetConsumer> consumer, + IntegrationHolder holder) { IntegrationProcessorStats stats = statsService - .map(svc -> svc.createIntegrationProcessorStats(integrationHolder.getIntegrationUuid())) + .map(svc -> svc.createIntegrationProcessorStats(holder.getIntegrationUuid())) .orElse(null); + processQueue(consumer, holder, this::dispatchMessage, stats, "messages", + pollDuration, packProcessingTimeout, + () -> ackStrategyFactory.newInstance(holder.getIntegrationId())); + } - while (isProcessorActive(integrationHolder)) { + void dispatchMessage(IntegrationHolder holder, UUID packetId, PublishIntegrationMsgProto msg, + IntegrationPackProcessingContext ctx) { + holder.getIntegration().process(msg, new BaseIntegrationMsgCallback(packetId, ctx)); + } + + /** + * Single consume/ack/commit loop shared by the data ({@link PublishIntegrationMsgProto}) and lifecycle-event + * ({@link ClientLifecycleEventMsgProto}) streams. Parameterized by the proto type, the per-message dispatcher, + * an optional per-stream stats sink (data stream and lifecycle-event stream register under distinct meter keys), + * a {@code kind} label used purely for logging, and the per-stream + * poll interval, pack-processing timeout, and ack-strategy supplier (data and events are configured + * independently under {@code queue.integration-msg} / {@code queue.integration-event}). + */ + private void processQueue(TbQueueControlledOffsetConsumer> consumer, + IntegrationHolder holder, + IntegrationDispatcher dispatcher, + IntegrationProcessorStats stats, + String kind, + long pollDuration, + long packProcessingTimeout, + Supplier> ackStrategyProvider) { + final AtomicLong counter = new AtomicLong(0); + while (isProcessorActive(holder)) { try { - List> messages = consumer.poll(pollDuration); + List> messages = consumer.poll(pollDuration); if (messages.isEmpty()) { continue; } - IntegrationAckStrategy ackStrategy = ackStrategyFactory.newInstance(integrationHolder.getIntegrationId()); - IntegrationSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(integrationHolder.getIntegrationId()); + IntegrationAckStrategy ackStrategy = ackStrategyProvider.get(); + IntegrationSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(holder.getIntegrationId()); long packId = counter.incrementAndGet(); if (packId == MAX_VALUE) { counter.set(0); } - var pendingMsgMap = toPendingMsgMap(messages, packId); + var pendingMsgMap = toPendingMap(messages, packId); submitStrategy.init(pendingMsgMap); - while (isProcessorActive(integrationHolder)) { - IntegrationPackProcessingContext ctx = new IntegrationPackProcessingContext(integrationHolder.getIntegrationId(), submitStrategy.getPendingMap()); + while (isProcessorActive(holder)) { + IntegrationPackProcessingContext ctx = + new IntegrationPackProcessingContext<>(holder.getIntegrationId(), submitStrategy.getPendingMap()); int totalMsgCount = pendingMsgMap.size(); - submitStrategy.process(entry -> integrationHolder.getIntegration().process(entry.getValue(), new BaseIntegrationMsgCallback(entry.getKey(), ctx))); + submitStrategy.process(entry -> dispatcher.dispatch(holder, entry.getKey(), entry.getValue(), ctx)); - if (isProcessorActive(integrationHolder)) { + if (isProcessorActive(holder)) { ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS); } - IntegrationPackProcessingResult result = new IntegrationPackProcessingResult(ctx); + IntegrationPackProcessingResult result = new IntegrationPackProcessingResult<>(ctx); ctx.cleanup(); - IntegrationProcessingDecision decision = ackStrategy.analyze(result); + IntegrationProcessingDecision decision = ackStrategy.analyze(result); - if (stats != null) stats.log(totalMsgCount, result, decision.isCommit()); + if (stats != null) { + stats.log(totalMsgCount, result, decision.isCommit()); + } if (decision.isCommit()) { consumer.commitSync(); @@ -228,19 +343,19 @@ private void processMessages(TbQueueControlledOffsetConsumer toPendingMsgMap(List> msgs, long packId) { - Map map = Maps.newLinkedHashMapWithExpectedSize(msgs.size()); + private Map toPendingMap(List> msgs, long packId) { + Map map = Maps.newLinkedHashMapWithExpectedSize(msgs.size()); int i = 0; for (var msg : msgs) { - UUID id = new UUID(packId, i++); - map.put(id, msg.getValue()); + map.put(new UUID(packId, i++), msg.getValue()); } return map; } + + @FunctionalInterface + private interface IntegrationDispatcher { + void dispatch(IntegrationHolder holder, UUID packetId, T msg, IntegrationPackProcessingContext ctx); + } } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/BurstIntegrationSubmitStrategy.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/BurstIntegrationSubmitStrategy.java index c2b228f06..13db6ba4a 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/BurstIntegrationSubmitStrategy.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/BurstIntegrationSubmitStrategy.java @@ -18,7 +18,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import java.util.Map; import java.util.UUID; @@ -28,34 +27,34 @@ @Slf4j @RequiredArgsConstructor -public class BurstIntegrationSubmitStrategy implements IntegrationSubmitStrategy { +public class BurstIntegrationSubmitStrategy implements IntegrationSubmitStrategy { @Getter private final String integrationId; - private Map publishMsgMap; + private Map publishMsgMap; @Override - public void init(Map messages) { + public void init(Map messages) { log.debug("[{}] Init pack {}", integrationId, messages.size()); this.publishMsgMap = messages; } @Override - public ConcurrentMap getPendingMap() { + public ConcurrentMap getPendingMap() { return new ConcurrentHashMap<>(publishMsgMap); } @Override - public void process(Consumer> msgConsumer) { + public void process(Consumer> msgConsumer) { log.debug("[{}] Start sending the pack of messages {}", integrationId, publishMsgMap.size()); - for (Map.Entry entry : publishMsgMap.entrySet()) { + for (Map.Entry entry : publishMsgMap.entrySet()) { msgConsumer.accept(entry); } } @Override - public void update(Map reprocessMap) { + public void update(Map reprocessMap) { log.debug("[{}] Updating the pack of messages {}", integrationId, reprocessMap.size()); publishMsgMap = reprocessMap; } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategy.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategy.java index 03ac18c0e..6fe766456 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategy.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategy.java @@ -17,8 +17,8 @@ import org.thingsboard.mqtt.broker.integration.api.data.IntegrationPackProcessingResult; -public interface IntegrationAckStrategy { +public interface IntegrationAckStrategy { - IntegrationProcessingDecision analyze(IntegrationPackProcessingResult processingResult); + IntegrationProcessingDecision analyze(IntegrationPackProcessingResult processingResult); } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyConfiguration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyConfiguration.java index e8774a475..72d71d4de 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyConfiguration.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyConfiguration.java @@ -15,17 +15,16 @@ */ package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; -import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +/** + * Ack strategy for the data ({@code tbmq.msg.ie}) stream. Shares its shape with + * {@link IntegrationAckStrategyProperties}; see {@link IntegrationEventAckStrategyConfiguration} for the sibling + * lifecycle-event config. + */ @Component @ConfigurationProperties(prefix = "queue.integration-msg.ack-strategy") -@Data -public class IntegrationAckStrategyConfiguration { - - private IntegrationAckStrategyType type; - private int retries; - private int pauseBetweenRetries; +public class IntegrationAckStrategyConfiguration extends IntegrationAckStrategyProperties { } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactory.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactory.java index eeb3fb345..16034bef7 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactory.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactory.java @@ -18,14 +18,15 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import org.thingsboard.mqtt.broker.integration.api.data.IntegrationPackProcessingResult; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Component @@ -33,22 +34,32 @@ public class IntegrationAckStrategyFactory { private final IntegrationAckStrategyConfiguration ackStrategyConfiguration; + private final IntegrationEventAckStrategyConfiguration eventAckStrategyConfiguration; - public IntegrationAckStrategy newInstance(String integrationId) { - return switch (ackStrategyConfiguration.getType()) { - case SKIP_ALL -> new SkipStrategy(integrationId); - case RETRY_ALL -> - new RetryStrategy(integrationId, ackStrategyConfiguration.getRetries(), ackStrategyConfiguration.getPauseBetweenRetries()); + public IntegrationAckStrategy newInstance(String integrationId) { + return build(integrationId, ackStrategyConfiguration.getType(), + ackStrategyConfiguration.getRetries(), ackStrategyConfiguration.getPauseBetweenRetries()); + } + + public IntegrationAckStrategy newEventInstance(String integrationId) { + return build(integrationId, eventAckStrategyConfiguration.getType(), + eventAckStrategyConfiguration.getRetries(), eventAckStrategyConfiguration.getPauseBetweenRetries()); + } + + private IntegrationAckStrategy build(String integrationId, IntegrationAckStrategyType type, int retries, int pauseBetweenRetries) { + return switch (type) { + case SKIP_ALL -> new SkipStrategy<>(integrationId); + case RETRY_ALL -> new RetryStrategy<>(integrationId, retries, pauseBetweenRetries); }; } @RequiredArgsConstructor - private static class SkipStrategy implements IntegrationAckStrategy { + private static class SkipStrategy implements IntegrationAckStrategy { private final String integrationId; @Override - public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult result) { + public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult result) { if (!result.getPendingMap().isEmpty() || !result.getFailedMap().isEmpty()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip reprocess for {} failed and {} timeout messages.", integrationId, result.getFailedMap().size(), result.getPendingMap().size()); @@ -56,20 +67,18 @@ public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult res } if (log.isTraceEnabled()) { result.getFailedMap().forEach((packetId, msg) -> - log.trace("[{}] Failed message: id - {}, topic - {}.", - integrationId, msg.getPublishMsgProto().getPacketId(), msg.getPublishMsgProto().getTopicName()) + log.trace("[{}] Failed message: id - {}.", integrationId, packetId) ); result.getPendingMap().forEach((packetId, msg) -> - log.trace("[{}] Timeout message: id - {}, topic - {}.", - integrationId, msg.getPublishMsgProto().getPacketId(), msg.getPublishMsgProto().getTopicName()) + log.trace("[{}] Timeout message: id - {}.", integrationId, packetId) ); } - return new IntegrationProcessingDecision(true, Collections.emptyMap()); + return new IntegrationProcessingDecision<>(true, Collections.emptyMap()); } } @RequiredArgsConstructor - private static class RetryStrategy implements IntegrationAckStrategy { + private static class RetryStrategy implements IntegrationAckStrategy { private final String integrationId; private final int maxRetries; @@ -78,30 +87,31 @@ private static class RetryStrategy implements IntegrationAckStrategy { private int retryCount; @Override - public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult result) { - Map pendingMap = result.getPendingMap(); - Map failedMap = result.getFailedMap(); + public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult result) { + Map pendingMap = result.getPendingMap(); + Map failedMap = result.getFailedMap(); if (pendingMap.isEmpty() && failedMap.isEmpty()) { - return new IntegrationProcessingDecision(true, Collections.emptyMap()); + return new IntegrationProcessingDecision<>(true, Collections.emptyMap()); } if (maxRetries != 0 && ++retryCount > maxRetries) { log.debug("[{}] Skip reprocess due to max retries.", integrationId); - return new IntegrationProcessingDecision(true, Collections.emptyMap()); + return new IntegrationProcessingDecision<>(true, Collections.emptyMap()); } - Map toReprocess = new HashMap<>(); - toReprocess.putAll(pendingMap); - toReprocess.putAll(failedMap); + // The pending/failed maps are unordered (ConcurrentHashMap -> HashMap copies), but the keys are + // UUID(packId, index), so sorting by key restores the original consume/offset order for the retry pass. + Map toReprocess = Stream.concat(pendingMap.entrySet().stream(), failedMap.entrySet().stream()) + .sorted(Map.Entry.comparingByKey()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, + (a, b) -> a, LinkedHashMap::new)); if (log.isDebugEnabled()) { log.debug("[{}] Going to reprocess {} messages", integrationId, toReprocess.size()); } if (log.isTraceEnabled()) { failedMap.forEach((packetId, msg) -> - log.trace("[{}] Going to reprocess failed message: id - {}, topic - {}.", - integrationId, msg.getPublishMsgProto().getPacketId(), msg.getPublishMsgProto().getTopicName()) + log.trace("[{}] Going to reprocess failed message: id - {}.", integrationId, packetId) ); pendingMap.forEach((packetId, msg) -> - log.trace("[{}] Going to reprocess timed-out message: id - {}, topic - {}.", - integrationId, msg.getPublishMsgProto().getPacketId(), msg.getPublishMsgProto().getTopicName()) + log.trace("[{}] Going to reprocess timed-out message: id - {}.", integrationId, packetId) ); } if (pauseBetweenRetries > 0) { @@ -111,7 +121,7 @@ public IntegrationProcessingDecision analyze(IntegrationPackProcessingResult res log.error("[{}] Failed to pause for retry", integrationId); } } - return new IntegrationProcessingDecision(false, toReprocess); + return new IntegrationProcessingDecision<>(false, toReprocess); } } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyProperties.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyProperties.java new file mode 100644 index 000000000..6a701e4d0 --- /dev/null +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyProperties.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; + +import lombok.Getter; +import lombok.Setter; + +/** + * Shared ack-strategy properties for the integration-executor consume loops. Concrete subclasses only supply the + * {@code @ConfigurationProperties} prefix so the data and lifecycle-event streams bind independently while a new + * field is declared here once. Left abstract (no {@code @ConfigurationProperties}) so it is never bound as a bean + * on its own, keeping the two concrete configs unambiguous to inject by type. + */ +@Getter +@Setter +public abstract class IntegrationAckStrategyProperties { + + private IntegrationAckStrategyType type; + private int retries; + private int pauseBetweenRetries; + +} diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationEventAckStrategyConfiguration.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationEventAckStrategyConfiguration.java new file mode 100644 index 000000000..05def80dc --- /dev/null +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationEventAckStrategyConfiguration.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Ack strategy for the dedicated lifecycle-event stream. Kept as a distinct type (rather than a second + * binding of {@link IntegrationAckStrategyConfiguration}) so both configs inject unambiguously, and so events + * can stay best-effort ({@code SKIP_ALL}) independently of the data stream's strategy. The common shape lives in + * {@link IntegrationAckStrategyProperties}. + */ +@Component +@ConfigurationProperties(prefix = "queue.integration-event.ack-strategy") +public class IntegrationEventAckStrategyConfiguration extends IntegrationAckStrategyProperties { + +} diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationProcessingDecision.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationProcessingDecision.java index 4f0d1dfe3..ad24bde7f 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationProcessingDecision.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationProcessingDecision.java @@ -16,15 +16,14 @@ package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; import lombok.Data; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; import java.util.Map; import java.util.UUID; @Data -public class IntegrationProcessingDecision { +public class IntegrationProcessingDecision { private final boolean commit; - private final Map reprocessMap; + private final Map reprocessMap; } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategy.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategy.java index bc5090d63..02e672a63 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategy.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategy.java @@ -15,22 +15,20 @@ */ package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; -import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; - import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -public interface IntegrationSubmitStrategy { +public interface IntegrationSubmitStrategy { - void init(Map messages); + void init(Map messages); - ConcurrentMap getPendingMap(); + ConcurrentMap getPendingMap(); - void process(Consumer> msgConsumer); + void process(Consumer> msgConsumer); - void update(Map reprocessMap); + void update(Map reprocessMap); String getIntegrationId(); } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategyFactory.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategyFactory.java index cd3ed77e0..2ea5ad5ab 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategyFactory.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationSubmitStrategyFactory.java @@ -22,8 +22,8 @@ @Slf4j public class IntegrationSubmitStrategyFactory { - public IntegrationSubmitStrategy newInstance(String integrationId) { - return new BurstIntegrationSubmitStrategy(integrationId); + public IntegrationSubmitStrategy newInstance(String integrationId) { + return new BurstIntegrationSubmitStrategy<>(integrationId); } } diff --git a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/callback/BaseIntegrationMsgCallback.java b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/callback/BaseIntegrationMsgCallback.java index 36a5eff2d..fa3e01e2e 100644 --- a/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/callback/BaseIntegrationMsgCallback.java +++ b/integration/executor/src/main/java/org/thingsboard/mqtt/broker/integration/service/processing/callback/BaseIntegrationMsgCallback.java @@ -27,7 +27,7 @@ public class BaseIntegrationMsgCallback implements IntegrationMsgCallback { private final UUID id; - private final IntegrationPackProcessingContext ctx; + private final IntegrationPackProcessingContext ctx; @Override public void onSuccess() { diff --git a/integration/executor/src/main/resources/tbmq-integration-executor.yml b/integration/executor/src/main/resources/tbmq-integration-executor.yml index 008794927..18c9f52a5 100644 --- a/integration/executor/src/main/resources/tbmq-integration-executor.yml +++ b/integration/executor/src/main/resources/tbmq-integration-executor.yml @@ -38,6 +38,20 @@ queue: retries: "${TB_IE_MSG_ACK_STRATEGY_RETRIES:5}" # Time to wait in the consumer thread before retrying (in s). pause-between-retries: "${TB_IE_MSG_ACK_STRATEGY_PAUSE_BETWEEN_RETRIES:1}" + integration-event: + # Poll interval for the dedicated 'tbmq.ie.event' lifecycle-event topics (in ms). + poll-interval: "${TB_IE_EVENT_MSG_POLL_INTERVAL:1000}" + # Timeout for processing a pack of lifecycle events (in ms). + pack-processing-timeout: "${TB_IE_EVENT_MSG_PACK_PROCESSING_TIMEOUT:30000}" + ack-strategy: + # Processing strategy for 'tbmq.ie.event' topics. Accepted values: SKIP_ALL, RETRY_ALL. + # Defaults to SKIP_ALL because lifecycle events are best-effort hints - retrying would redeliver + # stale events out of order; kept independent of the data stream, so it is not coupled to RETRY_ALL. + type: "${TB_IE_EVENT_MSG_ACK_STRATEGY_TYPE:SKIP_ALL}" + # Number of retries. Use 0 for unlimited. Applies to the RETRY_ALL strategy. + retries: "${TB_IE_EVENT_MSG_ACK_STRATEGY_RETRIES:5}" + # Time to wait in the consumer thread before retrying (in s). + pause-between-retries: "${TB_IE_EVENT_MSG_ACK_STRATEGY_PAUSE_BETWEEN_RETRIES:1}" kafka: # List of Kafka bootstrap servers used to establish connection. @@ -158,6 +172,17 @@ queue: additional-consumer-config: "${TB_KAFKA_IE_MSG_ADDITIONAL_CONSUMER_CONFIG:max.poll.records:50}" # Additional Kafka producer configs separated by semicolon for the `tbmq.msg.ie` topics. additional-producer-config: "${TB_KAFKA_IE_MSG_ADDITIONAL_PRODUCER_CONFIG:}" + integration-event: + # Kafka topic properties separated by semicolon for the dedicated `tbmq.ie.event` lifecycle-event topics. + # Retention defaults shorter than the data topic since events are best-effort hints. + topic-properties: "${TB_KAFKA_IE_EVENT_MSG_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:26214400;retention.bytes:104857600;partitions:1;replication.factor:1}" + # Additional Kafka consumer configs separated by semicolon for the `tbmq.ie.event` topics. + additional-consumer-config: "${TB_KAFKA_IE_EVENT_MSG_ADDITIONAL_CONSUMER_CONFIG:max.poll.records:50}" + # Additional Kafka producer configs separated by semicolon for the `tbmq.ie.event` topics. Lifecycle events are + # sent synchronously on the MQTT processing thread (same path as a regular publish), so a slow or unavailable Kafka + # can stall that thread for up to max.block.ms before the send fails, after which the event is dropped and counted. + # Lower max.block.ms to tighten that bound at the cost of more dropped events under Kafka pressure. + additional-producer-config: "${TB_KAFKA_IE_EVENT_MSG_ADDITIONAL_PRODUCER_CONFIG:max.block.ms:5000}" # Common prefix for all Kafka topics, producers, consumer groups, and consumers. Defaults to empty string, meaning no prefix is applied. kafka-prefix: "${TB_KAFKA_PREFIX:}" # Custom consumer configuration per Kafka topic. diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidatorTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidatorTest.java index ceb06727a..5207c20ab 100644 --- a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidatorTest.java +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttConfigValidatorTest.java @@ -167,6 +167,44 @@ void testNegativeKeepAliveThrowsException() { assertEquals("Keep Alive (seconds) must not be less than 0", exception.getMessage()); } + @Test + void testValidateEventsTopicWithStaticTopicDoesNotThrow() { + MqttIntegrationConfig config = createValidConfig(); + config.setEventsTopicName("events/topic"); + + assertDoesNotThrow(() -> MqttConfigValidator.validateEventsTopic(config)); + } + + @Test + void testValidateEventsTopicEmptyThrowsException() { + MqttIntegrationConfig config = createValidConfig(); + config.setEventsTopicName(""); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> + MqttConfigValidator.validateEventsTopic(config)); + assertEquals("Topic name is required to deliver lifecycle events", exception.getMessage()); + } + + @Test + void testValidateEventsTopicWildcardThrowsException() { + MqttIntegrationConfig config = createValidConfig(); + config.setEventsTopicName("abc/#"); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> + MqttConfigValidator.validateEventsTopic(config)); + assertEquals("Topic name cannot contain wildcard characters", exception.getMessage()); + } + + @Test + void testValidateEventsTopicDollarPrefixThrowsException() { + MqttIntegrationConfig config = createValidConfig(); + config.setEventsTopicName("$abc/abc"); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> + MqttConfigValidator.validateEventsTopic(config)); + assertEquals("Topic name cannot start with $ character", exception.getMessage()); + } + private MqttIntegrationConfig createValidConfig() { MqttIntegrationConfig config = new MqttIntegrationConfig(); config.setHost("mqtt.example.com"); diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationTest.java index 232509bdd..5a4b9aeb0 100644 --- a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationTest.java +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/integration/mqtt/MqttIntegrationTest.java @@ -15,23 +15,37 @@ */ package org.thingsboard.mqtt.broker.integration.service.integration.mqtt; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.util.concurrent.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.broker.common.data.exception.ThingsboardException; import org.thingsboard.mqtt.broker.common.data.integration.Integration; +import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; import org.thingsboard.mqtt.broker.common.util.JacksonUtil; import org.thingsboard.mqtt.broker.integration.api.IntegrationContext; import org.thingsboard.mqtt.broker.integration.api.TbIntegrationInitParams; import org.thingsboard.mqtt.broker.integration.api.callback.IntegrationMsgCallback; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class MqttIntegrationTest { @@ -74,6 +88,53 @@ void testDoValidateConfiguration_InvalidConfig() { assertThrows(ThingsboardException.class, () -> mqttIntegration.doValidateConfiguration(JacksonUtil.valueToTree(config), true)); } + @Test + void testValidateConfiguration_eventsEnabledWithEmptyEventsTopic_throws() { + config.setEventsTopicName(""); + IntegrationLifecycleMsg msg = lifecycleMsgWithEvents(config, "CLIENT_CONNECTED"); + + assertThrows(ThingsboardException.class, () -> mqttIntegration.validateConfiguration(msg, true)); + } + + @Test + void testValidateConfiguration_eventsOptedInViaNonArray_throwsOnEmptyEventsTopic() { + config.setEventsTopicName(""); + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.set("clientConfiguration", JacksonUtil.valueToTree(config)); + // a non-empty, non-array lifecycleEventTypes value: validation must still treat events as opted in, + // matching the runtime IntegrationEventsOptInUtil.isOptedIn predicate (present + non-empty) + configuration.putObject("lifecycleEventTypes").put("CLIENT_CONNECTED", true); + IntegrationLifecycleMsg msg = IntegrationLifecycleMsg.builder().configuration(configuration).build(); + + assertThrows(ThingsboardException.class, () -> mqttIntegration.validateConfiguration(msg, true)); + } + + @Test + void testValidateConfiguration_eventsEnabledWithValidEventsTopic_doesNotThrow() { + config.setEventsTopicName("tbmq/events"); + IntegrationLifecycleMsg msg = lifecycleMsgWithEvents(config, "CLIENT_CONNECTED"); + + assertDoesNotThrow(() -> mqttIntegration.validateConfiguration(msg, true)); + } + + @Test + void testValidateConfiguration_noEventsWithEmptyEventsTopic_doesNotThrow() { + config.setEventsTopicName(""); + IntegrationLifecycleMsg msg = lifecycleMsgWithEvents(config); // no event types + + assertDoesNotThrow(() -> mqttIntegration.validateConfiguration(msg, true)); + } + + private static IntegrationLifecycleMsg lifecycleMsgWithEvents(MqttIntegrationConfig config, String... eventTypes) { + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.set("clientConfiguration", JacksonUtil.valueToTree(config)); + ArrayNode types = configuration.putArray("lifecycleEventTypes"); + for (String type : eventTypes) { + types.add(type); + } + return IntegrationLifecycleMsg.builder().configuration(configuration).build(); + } + @Test void testDoCheckConnection_Failure() { Integration integration = new Integration(); @@ -86,4 +147,17 @@ void testDoStopClient() { verify(mockClient, times(1)).disconnect(); } + @Test + @SuppressWarnings("unchecked") + void testDoProcessLifecycleEvent_publishesToEventsTopicWithQos1AndNoRetain() { + config.setEventsTopicName("tbmq/events"); + ReflectionTestUtils.setField(mqttIntegration, "config", config); + Future future = mock(Future.class); + when(mockClient.publish(anyString(), any(ByteBuf.class), any(MqttQoS.class), anyBoolean())).thenReturn(future); + + mqttIntegration.doProcessLifecycleEvent(JacksonUtil.newObjectNode(), mockCallback); + + verify(mockClient).publish(eq("tbmq/events"), any(ByteBuf.class), eq(MqttQoS.AT_LEAST_ONCE), eq(false)); + } + } diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtilTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtilTest.java new file mode 100644 index 000000000..370e44782 --- /dev/null +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationEventsOptInUtilTest.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.jupiter.api.Test; +import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; +import org.thingsboard.mqtt.broker.common.util.JacksonUtil; + +import static org.assertj.core.api.Assertions.assertThat; + +class IntegrationEventsOptInUtilTest { + + private IntegrationLifecycleMsg msgWithConfig(String json) { + JsonNode config = json == null ? null : JacksonUtil.toJsonNode(json); + return IntegrationLifecycleMsg.builder().configuration(config).build(); + } + + @Test + void givenNonEmptyLifecycleEventTypes_whenIsOptedIn_thenTrue() { + var msg = msgWithConfig("{\"lifecycleEventTypes\":[\"CLIENT_CONNECTED\"]}"); + assertThat(IntegrationEventsOptInUtil.isOptedIn(msg)).isTrue(); + } + + @Test + void givenEmptyLifecycleEventTypesArray_whenIsOptedIn_thenFalse() { + var msg = msgWithConfig("{\"lifecycleEventTypes\":[]}"); + assertThat(IntegrationEventsOptInUtil.isOptedIn(msg)).isFalse(); + } + + @Test + void givenMissingLifecycleEventTypes_whenIsOptedIn_thenFalse() { + var msg = msgWithConfig("{\"topicFilters\":[\"a/b\"]}"); + assertThat(IntegrationEventsOptInUtil.isOptedIn(msg)).isFalse(); + } + + @Test + void givenNullConfiguration_whenIsOptedIn_thenFalse() { + assertThat(IntegrationEventsOptInUtil.isOptedIn(msgWithConfig(null))).isFalse(); + } + + @Test + void givenNullLifecycleMsg_whenIsOptedIn_thenFalse() { + assertThat(IntegrationEventsOptInUtil.isOptedIn(null)).isFalse(); + } +} diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsOptOutTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsOptOutTest.java new file mode 100644 index 000000000..245f5abfc --- /dev/null +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsOptOutTest.java @@ -0,0 +1,190 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; +import org.thingsboard.mqtt.broker.common.util.JacksonUtil; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; +import org.thingsboard.mqtt.broker.integration.api.TbPlatformIntegration; +import org.thingsboard.mqtt.broker.integration.service.processing.backpressure.IntegrationAckStrategyFactory; +import org.thingsboard.mqtt.broker.integration.service.processing.backpressure.IntegrationSubmitStrategyFactory; +import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.queue.provider.integration.IntegrationMsgQueueProvider; +import org.thingsboard.mqtt.broker.service.queue.IntegrationTopicService; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@Execution(ExecutionMode.SAME_THREAD) +class IntegrationMsgProcessorImplEventsOptOutTest { + + private static final String ID = "0198e1a0-aaaa-bbbb-cccc-ddddeeee0001"; + private static final String EVENT_TOPIC = "tbmq.ie.event.optout"; + private static final String EVENT_GROUP = "ie-event-cg-optout"; + + private IntegrationMsgQueueProvider queueProvider; + private IntegrationTopicService topicService; + private IntegrationMsgProcessorImpl processor; + + @BeforeEach + void setUp() { + queueProvider = mock(IntegrationMsgQueueProvider.class); + topicService = mock(IntegrationTopicService.class); + processor = new IntegrationMsgProcessorImpl( + queueProvider, topicService, + mock(IntegrationAckStrategyFactory.class), + mock(IntegrationSubmitStrategyFactory.class), + Optional.empty()); + processor.init(); + } + + @AfterEach + void tearDown() { + processor.destroy(); + } + + /** + * Simulates an integration UPDATE that removes lifecycleEventTypes (opt-out): + * 1) start with opt-in → events consumer is created + * 2) stop → events consumer is torn down + * 3) start with opt-out → events consumer must NOT be recreated + * Also verifies that a subsequent opt-in restart re-creates it (round-trip). + * Each "restart" uses a fresh integration mock (as AbstractIntegration.update creates a new init), + * but all share the same integrationId so the processor treats them as the same integration. + */ + @Test + void givenOptInThenOptOutRestart_whenStart_thenEventConsumerNotRecreated() { + // --- common topic stubs --- + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.optout"); + when(topicService.createEventTopic(ID)).thenReturn(EVENT_TOPIC); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-cg-optout"); + when(topicService.getEventConsumerGroup(ID)).thenReturn(EVENT_GROUP); + + var dataCons1 = controlledConsumer(); + var dataCons2 = controlledConsumer(); + var dataCons3 = controlledConsumer(); + var eventCons = controlledEventConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))) + .thenReturn(dataCons1, dataCons2, dataCons3); + when(queueProvider.getIeEventMsgConsumer(eq(EVENT_TOPIC), eq(EVENT_GROUP), eq(ID))) + .thenReturn(eventCons); + + // Step 1: start opt-in → events consumer created + TbPlatformIntegration integrationOptIn1 = integrationMock(ID, true); + processor.startProcessingIntegrationMessages(integrationOptIn1); + + verify(topicService, times(1)).createEventTopic(ID); + verify(queueProvider, times(1)).getIeEventMsgConsumer(EVENT_TOPIC, EVENT_GROUP, ID); + + // Step 2: stop → both data and events consumers are torn down + processor.stopProcessingIntegrationMessages(ID); + + // Step 3: restart with opt-out → NO new events consumer + TbPlatformIntegration integrationOptOut = integrationMock(ID, false); + processor.startProcessingIntegrationMessages(integrationOptOut); + + // createEventTopic and getIeEventMsgConsumer still called exactly once overall + verify(topicService, times(1)).createEventTopic(ID); + verify(queueProvider, times(1)).getIeEventMsgConsumer(anyString(), anyString(), anyString()); + + // Step 4 (round-trip): stop and restart with opt-in again → events consumer recreated + processor.stopProcessingIntegrationMessages(ID); + TbPlatformIntegration integrationOptIn2 = integrationMock(ID, true); + processor.startProcessingIntegrationMessages(integrationOptIn2); + + verify(topicService, times(2)).createEventTopic(ID); + verify(queueProvider, times(2)).getIeEventMsgConsumer(EVENT_TOPIC, EVENT_GROUP, ID); + } + + /** + * If createEventTopic throws (e.g. transient Kafka error), the data-path start + * must still complete successfully — events are best-effort. + */ + @Test + void givenCreateEventTopicThrows_whenStart_thenDataStartNotAffected() { + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.optout"); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-cg-optout"); + when(topicService.createEventTopic(ID)).thenThrow(new RuntimeException("kafka down")); + + var dataCons = controlledConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))).thenReturn(dataCons); + + TbPlatformIntegration integration = integrationMock(ID, true); + assertDoesNotThrow(() -> processor.startProcessingIntegrationMessages(integration)); + + // data consumer must still be created despite the events topic failure + verify(queueProvider).getIeMsgConsumer(anyString(), anyString(), eq(ID)); + } + + // ---- helpers ---- + + private TbPlatformIntegration integrationMock(String id, boolean optIn) { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(id); + when(integration.getLifecycleMsg()).thenReturn(optedIn(optIn)); + return integration; + } + + private IntegrationLifecycleMsg optedIn(boolean optIn) { + String json = optIn + ? "{\"lifecycleEventTypes\":[\"CLIENT_CONNECTED\"]}" + : "{\"topicFilters\":[\"a/b\"]}"; + return IntegrationLifecycleMsg.builder() + .integrationId(UUID.fromString(ID)) + .name("ie-optout") + .configuration(JacksonUtil.toJsonNode(json)) + .build(); + } + + @SuppressWarnings("unchecked") + private TbQueueControlledOffsetConsumer> controlledConsumer() { + TbQueueControlledOffsetConsumer> c = + mock(TbQueueControlledOffsetConsumer.class); + doReturn("tbmq.msg.ie.optout").when(c).getTopic(); + doReturn(Optional.of(0L)).when(c).getCommittedOffset(anyString(), anyInt()); + doReturn(List.of()).when(c).poll(anyLong()); + return c; + } + + @SuppressWarnings("unchecked") + private TbQueueControlledOffsetConsumer> controlledEventConsumer() { + TbQueueControlledOffsetConsumer> c = + mock(TbQueueControlledOffsetConsumer.class); + doReturn(EVENT_TOPIC).when(c).getTopic(); + doReturn(Optional.of(0L)).when(c).getCommittedOffset(anyString(), anyInt()); + doReturn(List.of()).when(c).poll(anyLong()); + return c; + } +} diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsTest.java new file mode 100644 index 000000000..1c4dbf6dd --- /dev/null +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/IntegrationMsgProcessorImplEventsTest.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.thingsboard.mqtt.broker.common.data.integration.IntegrationLifecycleMsg; +import org.thingsboard.mqtt.broker.common.util.JacksonUtil; +import org.thingsboard.mqtt.broker.gen.integration.ClientLifecycleEventMsgProto; +import org.thingsboard.mqtt.broker.gen.integration.PublishIntegrationMsgProto; +import org.thingsboard.mqtt.broker.integration.api.IntegrationStatisticsService; +import org.thingsboard.mqtt.broker.integration.api.TbPlatformIntegration; +import org.thingsboard.mqtt.broker.integration.api.callback.IntegrationMsgCallback; +import org.thingsboard.mqtt.broker.integration.api.data.IntegrationPackProcessingContext; +import org.thingsboard.mqtt.broker.integration.service.data.IntegrationHolder; +import org.thingsboard.mqtt.broker.integration.service.processing.backpressure.IntegrationAckStrategyFactory; +import org.thingsboard.mqtt.broker.integration.service.processing.backpressure.IntegrationSubmitStrategyFactory; +import org.thingsboard.mqtt.broker.queue.TbQueueControlledOffsetConsumer; +import org.thingsboard.mqtt.broker.queue.common.TbProtoQueueMsg; +import org.thingsboard.mqtt.broker.queue.provider.integration.IntegrationMsgQueueProvider; +import org.thingsboard.mqtt.broker.service.queue.IntegrationTopicService; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@Execution(ExecutionMode.SAME_THREAD) +class IntegrationMsgProcessorImplEventsTest { + + private static final String ID = "0198e1a0-1111-2222-3333-444455556666"; + private static final String EVENT_TOPIC = "tbmq.ie.event.x"; + private static final String EVENT_GROUP = "ie-event-consumer-group-x"; + + private IntegrationMsgQueueProvider queueProvider; + private IntegrationTopicService topicService; + private IntegrationMsgProcessorImpl processor; + + @BeforeEach + void setUp() { + queueProvider = mock(IntegrationMsgQueueProvider.class); + topicService = mock(IntegrationTopicService.class); + processor = new IntegrationMsgProcessorImpl( + queueProvider, topicService, + mock(IntegrationAckStrategyFactory.class), + mock(IntegrationSubmitStrategyFactory.class), + Optional.empty()); + processor.init(); + } + + @AfterEach + void tearDown() { + processor.destroy(); + } + + @Test + void givenPolledLifecycleEvent_whenDispatchEvent_thenRoutedToProcessLifecycleEvent() { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(ID); + IntegrationHolder holder = new IntegrationHolder(integration); + + ClientLifecycleEventMsgProto event = ClientLifecycleEventMsgProto.newBuilder() + .setEventType("CLIENT_CONNECTED").setClientId("c1").build(); + UUID packetId = new UUID(1L, 0L); + IntegrationPackProcessingContext ctx = + new IntegrationPackProcessingContext<>(ID, new ConcurrentHashMap<>()); + + processor.dispatchEvent(holder, packetId, event, ctx); + + verify(integration).processLifecycleEvent(eq(event), any(IntegrationMsgCallback.class)); + } + + @Test + void givenOptedInIntegration_whenStart_thenCreatesEventTopicAndConsumer() { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(ID); + when(integration.getLifecycleMsg()).thenReturn(optedIn(true)); + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.x"); + when(topicService.createEventTopic(ID)).thenReturn(EVENT_TOPIC); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-consumer-group-x"); + when(topicService.getEventConsumerGroup(ID)).thenReturn(EVENT_GROUP); + var dataCons = controlledConsumer(); + var eventCons = controlledEventConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))).thenReturn(dataCons); + when(queueProvider.getIeEventMsgConsumer(eq(EVENT_TOPIC), eq(EVENT_GROUP), eq(ID))).thenReturn(eventCons); + + processor.startProcessingIntegrationMessages(integration); + + verify(topicService, timeout(2000)).createEventTopic(ID); + verify(queueProvider, timeout(2000)).getIeEventMsgConsumer(EVENT_TOPIC, EVENT_GROUP, ID); + } + + @Test + void givenOptedOutIntegration_whenStart_thenNoEventTopicNorConsumer() { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(ID); + when(integration.getLifecycleMsg()).thenReturn(optedIn(false)); + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.x"); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-consumer-group-x"); + var dataCons = controlledConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))).thenReturn(dataCons); + + processor.startProcessingIntegrationMessages(integration); + + verify(topicService, never()).createEventTopic(anyString()); + verify(queueProvider, never()).getIeEventMsgConsumer(anyString(), anyString(), anyString()); + } + + @Test + void givenStartedEvents_whenStop_thenEventConsumerCancelled() { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(ID); + when(integration.getLifecycleMsg()).thenReturn(optedIn(true)); + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.x"); + when(topicService.createEventTopic(ID)).thenReturn(EVENT_TOPIC); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-consumer-group-x"); + when(topicService.getEventConsumerGroup(ID)).thenReturn(EVENT_GROUP); + var dataCons = controlledConsumer(); + var eventCons = controlledEventConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))).thenReturn(dataCons); + when(queueProvider.getIeEventMsgConsumer(eq(EVENT_TOPIC), eq(EVENT_GROUP), eq(ID))).thenReturn(eventCons); + + processor.startProcessingIntegrationMessages(integration); + verify(eventCons, timeout(2000).atLeastOnce()).poll(anyLong()); + + processor.stopProcessingIntegrationMessages(ID); + + verify(eventCons, timeout(2000)).unsubscribeAndClose(); + } + + @Test + void givenStatsService_whenEventsStart_thenEventProcessorStatsCreated_andClearedOnStop() { + IntegrationStatisticsService statsService = mock(IntegrationStatisticsService.class); + IntegrationMsgProcessorImpl statsProcessor = new IntegrationMsgProcessorImpl( + queueProvider, topicService, + mock(IntegrationAckStrategyFactory.class), + mock(IntegrationSubmitStrategyFactory.class), + Optional.of(statsService)); + statsProcessor.init(); + try { + TbPlatformIntegration integration = mock(TbPlatformIntegration.class); + when(integration.getIntegrationId()).thenReturn(ID); + when(integration.getIntegrationUuid()).thenReturn(UUID.fromString(ID)); + when(integration.getLifecycleMsg()).thenReturn(optedIn(true)); + when(topicService.createTopic(ID)).thenReturn("tbmq.msg.ie.x"); + when(topicService.createEventTopic(ID)).thenReturn(EVENT_TOPIC); + when(topicService.getConsumerGroup(ID)).thenReturn("ie-msg-consumer-group-x"); + when(topicService.getEventConsumerGroup(ID)).thenReturn(EVENT_GROUP); + var dataCons = controlledConsumer(); + var eventCons = controlledEventConsumer(); + when(queueProvider.getIeMsgConsumer(anyString(), anyString(), eq(ID))).thenReturn(dataCons); + when(queueProvider.getIeEventMsgConsumer(eq(EVENT_TOPIC), eq(EVENT_GROUP), eq(ID))).thenReturn(eventCons); + + statsProcessor.startProcessingIntegrationMessages(integration); + + verify(statsService, timeout(2000)).createIntegrationProcessorStats(UUID.fromString(ID)); + verify(statsService, timeout(2000)).createIntegrationEventProcessorStats(UUID.fromString(ID)); + + statsProcessor.stopProcessingIntegrationMessages(ID); + + verify(statsService, timeout(2000)).clearIntegrationProcessorStats(UUID.fromString(ID)); + verify(statsService, timeout(2000)).clearIntegrationEventProcessorStats(UUID.fromString(ID)); + } finally { + statsProcessor.destroy(); + } + } + + // ---- helpers ---- + + private IntegrationLifecycleMsg optedIn(boolean optIn) { + String json = optIn + ? "{\"lifecycleEventTypes\":[\"CLIENT_CONNECTED\"]}" + : "{\"topicFilters\":[\"a/b\"]}"; + return IntegrationLifecycleMsg.builder() + .integrationId(UUID.fromString(ID)) + .name("ie") + .configuration(JacksonUtil.toJsonNode(json)) + .build(); + } + + @SuppressWarnings("unchecked") + private TbQueueControlledOffsetConsumer> controlledConsumer() { + TbQueueControlledOffsetConsumer> c = + mock(TbQueueControlledOffsetConsumer.class); + doReturn("tbmq.msg.ie.x").when(c).getTopic(); + doReturn(Optional.of(0L)).when(c).getCommittedOffset(anyString(), anyInt()); + doReturn(List.of()).when(c).poll(anyLong()); + return c; + } + + @SuppressWarnings("unchecked") + private TbQueueControlledOffsetConsumer> controlledEventConsumer() { + TbQueueControlledOffsetConsumer> c = + mock(TbQueueControlledOffsetConsumer.class); + doReturn(EVENT_TOPIC).when(c).getTopic(); + doReturn(Optional.of(0L)).when(c).getCommittedOffset(anyString(), anyInt()); + doReturn(List.of()).when(c).poll(anyLong()); + return c; + } +} diff --git a/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactoryTest.java b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactoryTest.java new file mode 100644 index 000000000..a513fe032 --- /dev/null +++ b/integration/executor/src/test/java/org/thingsboard/mqtt/broker/integration/service/processing/backpressure/IntegrationAckStrategyFactoryTest.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt.broker.integration.service.processing.backpressure; + +import org.junit.jupiter.api.Test; +import org.thingsboard.mqtt.broker.integration.api.data.IntegrationPackProcessingContext; +import org.thingsboard.mqtt.broker.integration.api.data.IntegrationPackProcessingResult; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class IntegrationAckStrategyFactoryTest { + + @Test + void givenDataRetryAndEventSkip_whenAnalyzeUnprocessedPack_thenDataReprocessesAndEventCommits() { + IntegrationAckStrategyConfiguration dataConfig = new IntegrationAckStrategyConfiguration(); + dataConfig.setType(IntegrationAckStrategyType.RETRY_ALL); + dataConfig.setRetries(1); + dataConfig.setPauseBetweenRetries(0); + + IntegrationEventAckStrategyConfiguration eventConfig = new IntegrationEventAckStrategyConfiguration(); + eventConfig.setType(IntegrationAckStrategyType.SKIP_ALL); + + IntegrationAckStrategyFactory factory = new IntegrationAckStrategyFactory(dataConfig, eventConfig); + + // data stream (RETRY_ALL) must NOT commit an unprocessed pack - it reprocesses... + assertFalse(factory.newInstance("id").analyze(resultWithPending()).isCommit()); + // ...while the events stream (SKIP_ALL) commits (drops) it, independent of the data strategy. + assertTrue(factory.newEventInstance("id").analyze(resultWithPending()).isCommit()); + } + + @Test + void givenUnorderedPendingAndFailed_whenRetryAnalyze_thenReprocessMapPreservesOffsetOrder() { + IntegrationAckStrategyConfiguration dataConfig = new IntegrationAckStrategyConfiguration(); + dataConfig.setType(IntegrationAckStrategyType.RETRY_ALL); + dataConfig.setRetries(1); + dataConfig.setPauseBetweenRetries(0); + + IntegrationAckStrategyFactory factory = + new IntegrationAckStrategyFactory(dataConfig, new IntegrationEventAckStrategyConfiguration()); + + // Keys are UUID(packId, offsetIndex); insert in shuffled order into an unordered ConcurrentHashMap. + long packId = 7L; + ConcurrentMap pending = new ConcurrentHashMap<>(); + for (int i : new int[]{3, 1, 5, 0, 4, 2}) { + pending.put(new UUID(packId, i), "m" + i); + } + IntegrationPackProcessingContext ctx = new IntegrationPackProcessingContext<>("id", pending); + // Move a couple of entries into the failed map so reprocess spans both pending and failed. + ctx.onFailure(new UUID(packId, 1)); + ctx.onFailure(new UUID(packId, 0)); + + IntegrationProcessingDecision decision = + factory.newInstance("id").analyze(new IntegrationPackProcessingResult<>(ctx)); + + assertFalse(decision.isCommit()); + List reprocessOrder = decision.getReprocessMap().keySet().stream() + .map(UUID::getLeastSignificantBits) + .toList(); + assertIterableEquals(List.of(0L, 1L, 2L, 3L, 4L, 5L), reprocessOrder); + } + + private static IntegrationPackProcessingResult resultWithPending() { + ConcurrentMap pending = new ConcurrentHashMap<>(); + pending.put(UUID.randomUUID(), new Object()); + return new IntegrationPackProcessingResult<>(new IntegrationPackProcessingContext<>("id", pending)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.html b/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.html index b80c44c27..85ab23a31 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.html +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.html @@ -19,6 +19,10 @@
@if (!isNew) { + + @if (baseHttpIntegrationConfigForm.hasError('topicFilterOrEventRequired')) { + {{ 'integration.topic-filter-or-event-required' | translate }} + } }
@@ -150,10 +154,21 @@
+ +
+
+

integration.lifecycle-events

+ + +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.ts b/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.ts index 701562f9d..d3821cfec 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/http-integration-form/http-integration-form.component.ts @@ -27,18 +27,21 @@ import { } from '@angular/forms'; import { baseUrl, isDefinedAndNotNull, notOnlyWhitespaceValidator } from '@core/utils'; import { takeUntil } from 'rxjs/operators'; -import { HttpIntegration, HttpRequestType, Integration } from '@shared/models/integration.models'; +import { atLeastOneFilterOrEvent, HttpIntegration, HttpRequestType, Integration } from '@shared/models/integration.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { TranslateModule, TranslateService } from '@ngx-translate/core'; import { IntegrationForm } from '@home/components/integration/configuration/integration-form'; import { IntegrationCredentialType } from '@shared/models/integration.models'; -import { MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; +import { MatError, MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; import { CopyButtonComponent } from '@shared/components/button/copy-button.component'; import { MatOption, MatSelect } from '@angular/material/select'; import { IntegrationTopicFiltersComponent } from '@home/components/integration/integration-topic-filters/integration-topic-filters.component'; +import { + IntegrationLifecycleEventsComponent +} from '@home/components/integration/lifecycle-events/integration-lifecycle-events.component'; import { IntegrationCredentialsComponent } from '@home/components/integration/integration-credentials/integration-credentials.component'; @@ -64,10 +67,12 @@ import { KeyValMapComponent } from '@shared/components/key-val-map.component'; imports: [ ReactiveFormsModule, MatFormField, + MatError, CopyButtonComponent, MatSelect, MatOption, IntegrationTopicFiltersComponent, + IntegrationLifecycleEventsComponent, IntegrationCredentialsComponent, TranslateModule, MatExpansionPanel, @@ -124,7 +129,8 @@ export class HttpIntegrationFormComponent extends IntegrationForm implements Con ngOnInit() { this.baseHttpIntegrationConfigForm = this.fb.group({ - topicFilters: [['tbmq/#'], Validators.required], + topicFilters: [['tbmq/#']], + lifecycleEventTypes: [[]], clientConfiguration: this.fb.group({ sendOnlyMsgPayload: [false, []], restEndpointUrl: [baseUrl(), [Validators.required, notOnlyWhitespaceValidator]], @@ -137,7 +143,7 @@ export class HttpIntegrationFormComponent extends IntegrationForm implements Con payloadContentType: [ContentType.BINARY, []], sendBinaryOnParseFailure: [true, []], }) - }); + }, {validators: atLeastOneFilterOrEvent}); this.baseHttpIntegrationConfigForm.valueChanges .pipe(takeUntil(this.destroy$)) .subscribe(() => this.updateModels(this.baseHttpIntegrationConfigForm.getRawValue())); @@ -181,6 +187,7 @@ export class HttpIntegrationFormComponent extends IntegrationForm implements Con private updateModels(value) { if (this.isNew) { delete value.topicFilters; + delete value.lifecycleEventTypes; } this.propagateChange(value); } diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.html b/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.html index f10f4c655..a598ae174 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.html +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.html @@ -19,6 +19,10 @@
@if (!isNew) { + + @if (kafkaIntegrationConfigForm.hasError('topicFilterOrEventRequired')) { + {{ 'integration.topic-filter-or-event-required' | translate }} + } }
@@ -206,10 +210,21 @@
+ +
+
+

integration.lifecycle-events

+ + +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.ts b/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.ts index 702540045..e49b0d2f7 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/kafka-integration-form/kafka-integration-form.component.ts @@ -29,12 +29,13 @@ import { isDefinedAndNotNull, notOnlyWhitespaceValidator } from '@core/utils'; import { takeUntil } from 'rxjs/operators'; import { IntegrationForm } from '@home/components/integration/configuration/integration-form'; import { + atLeastOneFilterOrEvent, Integration, KafkaIntegration, ToByteStandartCharsetTypes, ToByteStandartCharsetTypeTranslations } from '@shared/models/integration.models'; -import { MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; +import { MatError, MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; import { MatInput } from '@angular/material/input'; import { NgTemplateOutlet } from '@angular/common'; import { TranslateModule } from '@ngx-translate/core'; @@ -51,6 +52,9 @@ import { MatTooltip } from '@angular/material/tooltip'; import { IntegrationTopicFiltersComponent } from '@home/components/integration/integration-topic-filters/integration-topic-filters.component'; +import { + IntegrationLifecycleEventsComponent +} from '@home/components/integration/lifecycle-events/integration-lifecycle-events.component'; import { MatSlideToggle } from '@angular/material/slide-toggle'; import { CopyButtonComponent } from '@shared/components/button/copy-button.component'; import { HintTooltipIconComponent } from '@shared/components/hint-tooltip-icon.component'; @@ -62,6 +66,7 @@ import { HintTooltipIconComponent } from '@shared/components/hint-tooltip-icon.c imports: [ ReactiveFormsModule, MatFormField, + MatError, MatInput, MatLabel, TranslateModule, @@ -77,6 +82,7 @@ import { HintTooltipIconComponent } from '@shared/components/hint-tooltip-icon.c MatIcon, MatTooltip, IntegrationTopicFiltersComponent, + IntegrationLifecycleEventsComponent, MatSlideToggle, CopyButtonComponent, HintTooltipIconComponent @@ -117,7 +123,8 @@ export class KafkaIntegrationFormComponent extends IntegrationForm implements Co ngOnInit() { this.kafkaIntegrationConfigForm = this.fb.group({ - topicFilters: [['tbmq/#'], Validators.required], + topicFilters: [['tbmq/#']], + lifecycleEventTypes: [[]], clientConfiguration: this.fb.group({ sendOnlyMsgPayload: [false, []], topic: ['tbmq.messages', [Validators.required]], @@ -136,7 +143,7 @@ export class KafkaIntegrationFormComponent extends IntegrationForm implements Co kafkaHeaders: [null, []], kafkaHeadersCharset: ['UTF-8', []], }) - }); + }, {validators: atLeastOneFilterOrEvent}); this.kafkaIntegrationConfigForm.valueChanges .pipe(takeUntil(this.destroy$)) .subscribe(() => this.updateModels(this.kafkaIntegrationConfigForm.getRawValue())); @@ -176,6 +183,7 @@ export class KafkaIntegrationFormComponent extends IntegrationForm implements Co private updateModels(value) { if (this.isNew) { delete value.topicFilters; + delete value.lifecycleEventTypes; } this.propagateChange(value); } diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.html b/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.html index 99d0c65de..00d290d08 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.html +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.html @@ -19,6 +19,10 @@
@if (!isNew) { + + @if (mqttIntegrationConfigForm.hasError('topicFilterOrEventRequired')) { + {{ 'integration.topic-filter-or-event-required' | translate }} + } }
@@ -284,10 +288,48 @@
+ +
+
+

integration.lifecycle-events

+ + + @if (eventsEnabled) { +
+ + integration.events-topic-name + + + @for (option of filteredEventsTopics | async; track option) { + {{option}} + } + + + + + warning + + + help + + +
+ } +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.ts b/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.ts index 750852718..54f07b7d1 100644 --- a/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/integration/configuration/mqtt-integration-form/mqtt-integration-form.component.ts @@ -29,10 +29,11 @@ import { filterTopics, isDefinedAndNotNull, notOnlyWhitespaceValidator } from '@ import { map, takeUntil } from 'rxjs/operators'; import { IntegrationForm } from '@home/components/integration/configuration/integration-form'; import { + atLeastOneFilterOrEvent, Integration, MqttIntegration, } from '@shared/models/integration.models'; -import { MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; +import { MatError, MatFormField, MatLabel, MatSuffix } from '@angular/material/form-field'; import { MatInput } from '@angular/material/input'; import { AsyncPipe, NgTemplateOutlet } from '@angular/common'; import { TranslateModule } from '@ngx-translate/core'; @@ -48,6 +49,9 @@ import { MatTooltip } from '@angular/material/tooltip'; import { IntegrationTopicFiltersComponent } from '@home/components/integration/integration-topic-filters/integration-topic-filters.component'; +import { + IntegrationLifecycleEventsComponent +} from '@home/components/integration/lifecycle-events/integration-lifecycle-events.component'; import { MatIconButton } from '@angular/material/button'; import { clientIdRandom } from '@shared/models/ws-client.model'; import { @@ -70,6 +74,7 @@ import { Observable } from 'rxjs'; imports: [ ReactiveFormsModule, MatFormField, + MatError, MatInput, MatLabel, TranslateModule, @@ -84,6 +89,7 @@ import { Observable } from 'rxjs'; MatIcon, MatTooltip, IntegrationTopicFiltersComponent, + IntegrationLifecycleEventsComponent, MatIconButton, IntegrationCredentialsComponent, QosSelectComponent, @@ -115,6 +121,7 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con IntegrationCredentialType = IntegrationCredentialType; mqttVersions = MqttVersions; filteredTopics: Observable; + filteredEventsTopics: Observable; private propagateChangePending = false; private propagateChange = (v: any) => { }; @@ -123,19 +130,26 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con return this.mqttIntegrationConfigForm.get('clientConfiguration') as UntypedFormGroup; } + get eventsEnabled(): boolean { + const types = this.mqttIntegrationConfigForm.get('lifecycleEventTypes')?.value; + return Array.isArray(types) && types.length > 0; + } + constructor(private fb: UntypedFormBuilder) { super(); } ngOnInit() { this.mqttIntegrationConfigForm = this.fb.group({ - topicFilters: [['tbmq/#'], Validators.required], + topicFilters: [['tbmq/#']], + lifecycleEventTypes: [[]], clientConfiguration: this.fb.group({ sendOnlyMsgPayload: [false, []], host: [null, [Validators.required, notOnlyWhitespaceValidator]], port: [1883, [Validators.min(1), Validators.max(65535), Validators.pattern('[0-9]*'), Validators.required]], topicName: ['tbmq/messages', [Validators.required]], useMsgTopicName: [true, []], + eventsTopicName: ['tbmq/events', []], clientId: [clientIdRandom(), [Validators.required]], credentials: [{ type: IntegrationCredentialType.Anonymous }], ssl: [false, [Validators.required]], @@ -148,7 +162,7 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con useMsgRetain: [true, []], keepAliveSec: [60, [Validators.required]], }) - }); + }, {validators: atLeastOneFilterOrEvent}); this.initFormListeners(); } @@ -187,6 +201,7 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con private updateModels(value) { if (this.isNew) { delete value.topicFilters; + delete value.lifecycleEventTypes; } this.propagateChange(value); } @@ -252,6 +267,10 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con } }); + this.mqttIntegrationConfigForm.get('lifecycleEventTypes').valueChanges + .pipe(takeUntil(this.destroy$)) + .subscribe(() => this.updateEventsTopicState()); + setTimeout(() => { if (this.isNew) { this.clientConfigurationFormGroup.get('topicName').disable(); @@ -264,6 +283,11 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con takeUntil(this.destroy$), map(value => filterTopics(value || '')) ); + + this.filteredEventsTopics = this.clientConfigurationFormGroup.get('eventsTopicName').valueChanges.pipe( + takeUntil(this.destroy$), + map(value => filterTopics(value || '')) + ); } private updateView(value: MqttIntegration) { @@ -279,5 +303,19 @@ export class MqttIntegrationFormComponent extends IntegrationForm implements Con this.clientConfigurationFormGroup.get('retained').disable({emitEvent: false}); this.clientConfigurationFormGroup.get('retained').updateValueAndValidity({emitEvent: false}); } + this.updateEventsTopicState(); + } + + private updateEventsTopicState() { + if (this.disabled) { + return; + } + const control = this.clientConfigurationFormGroup.get('eventsTopicName'); + if (this.eventsEnabled) { + control.setValidators(Validators.required); + } else { + control.clearValidators(); + } + control.updateValueAndValidity({emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/integration/integration-topic-filters/integration-topic-filters.component.html b/ui-ngx/src/app/modules/home/components/integration/integration-topic-filters/integration-topic-filters.component.html index c6dbd6d32..078ff34c5 100644 --- a/ui-ngx/src/app/modules/home/components/integration/integration-topic-filters/integration-topic-filters.component.html +++ b/ui-ngx/src/app/modules/home/components/integration/integration-topic-filters/integration-topic-filters.component.html @@ -149,7 +149,7 @@ warning -