Fluentbit integration for docker

docker/docker-compose-base.yml

@@ -4,6 +4,10 @@ services:
     mem_limit: 512m
     environment:
       - SPRING_PROFILES_ACTIVE=docker
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
     networks:
       - shared-network
     #depends_on:
@@ -15,6 +19,10 @@ services:
     mem_limit: 512m
     environment:
       - SPRING_PROFILES_ACTIVE=docker
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
     networks:
       - shared-network
     #depends_on:
@@ -28,6 +36,10 @@ services:
       - "8080:8080"
     environment:
       - SPRING_PROFILES_ACTIVE=docker
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
     networks:
       - shared-network
 
@@ -38,6 +50,10 @@ services:
       - "9000:9000"
     environment:
       - SPRING_PROFILES_ACTIVE=docker
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
     networks:
       - shared-network
 

docker/docker-compose-infra.yml

@@ -43,35 +43,10 @@ services:
     networks:
       - shared-network
 
-  prometheus:
-    image: prom/prometheus:latest
-    volumes:
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-      - prometheus-data:/prometheus
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-    ports:
-      - "9090:9090"
-    networks:
-      - shared-network
-
-  grafana:
-    image: grafana/grafana:latest
-    volumes:
-      - grafana-data:/var/lib/grafana
-    ports:
-      - "3000:3000"
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-    depends_on:
-      - prometheus
-    networks:
-      - shared-network
-
 networks:
   shared-network:
     driver: bridge
 volumes:
   prometheus-data:
-  grafana-data:
+  grafana-data:
+  loki-data:

docker/docker-compose-observability.yml

@@ -0,0 +1,64 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+    ports:
+      - "9090:9090"
+    networks:
+      - shared-network
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - grafana-data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    depends_on:
+      - prometheus
+    networks:
+      - shared-network
+
+  loki:
+    image: grafana/loki:3.0.0
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml
+      - loki-data:/loki
+    networks:
+      - shared-network
+
+  fluent-bit:
+    image: grafana/fluent-bit-plugin-loki:2.9.8
+    container_name: fluent-bit
+    ports:
+      - "24224:24224"
+    environment:
+      - LOKI_URL=http://loki:3100/loki/api/v1/push
+    volumes:
+      - ./fluent-bit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+    networks:
+      - shared-network
+  tempo:
+    image: grafana/tempo:1.5.0
+    command: -config.file /etc/tempo-config.yml
+    ports:
+      - "4317:4317"
+    volumes:
+      - ./tempo/tempo.yml:/etc/tempo-config.yml
+
+networks:
+  shared-network:
+    driver: bridge
+volumes:
+  prometheus-data:
+  grafana-data:
+  loki-data:

docker/fluent-bit/fluent-bit.conf

@@ -0,0 +1,15 @@
+[INPUT]
+    Name        forward
+    Listen      0.0.0.0
+    Port        24224
+[Output]
+    Name        grafana-loki
+    Match       *
+    Url         http://loki:3100/loki/api/v1/push
+    RemoveKeys  source,container_id
+    Labels      {job="fluent-bit"}
+    LabelKeys   container_name
+    BatchWait   1s
+    BatchSize   1001024
+    LineFormat  json
+    LogLevel    info

docker/fluent-bit/parsers.conf

@@ -0,0 +1,12 @@
+[PARSER]
+    Name         logback_parser
+    Format       regex
+    Regex        ^(?<time>[^ ]+ [^ ]+) \[(?<thread>[^\]]+)\] (?<level>[^ ]+) (?<logger>[^ ]+) - (?<message>.*)$
+
+#(?<time>[^ ]+ [^ ]+)	Captures timestamp (2025-03-09 12:29:12.769)
+#\[(?<thread>[^\]]+)\]	Captures thread name (parallel-1)
+#\[(?<trace_id>[^\]]+)\]	Captures traceId (abc123)
+#\[(?<span_id>[^\]]+)\]	Captures spanId (def456)
+#(?<level>[^ ]+)	Captures log level (INFO)
+#(?<logger>[^ ]+)	Captures logger class (com.example.ClassName)
+#(?<message>.*)$	Captures log message (Log message here)

docker/loki/loki-config.yaml

@@ -0,0 +1,31 @@
+auth_enabled: false
+server:
+  http_listen_port: 3100
+ingester:
+  lifecycler:
+    address: 0.0.0.0
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 5m
+  wal:
+    enabled: true
+    dir: /loki/wal
+schema_config:
+  configs:
+    - from: 2025-03-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+  filesystem:
+    directory: /loki/chunks
+compactor:
+  working_directory: /loki/compactor

microservices/course-composite-service/pom.xml

@@ -65,7 +65,11 @@
 			<groupId>io.micrometer</groupId>
 			<artifactId>micrometer-registry-prometheus</artifactId>
 		</dependency>
-
+		<!-- Logback Classic for Logging -->
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+		</dependency>
 
 		<dependency>
 			<groupId>org.springframework.boot</groupId>

microservices/course-composite-service/src/main/java/io/javatab/microservices/composite/course/MetricsController.java

@@ -4,17 +4,25 @@
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Timer;
 import jakarta.annotation.PostConstruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ResponseStatusException;
 
 
 import java.util.Random;
 
+
+/*
+* Just for manual test for metrics and errors which doesn't sit under security
+* */
 @RestController
 @RequestMapping("/api/metrics")
 public class MetricsController {
-
+    private final Logger logger = LoggerFactory.getLogger(MetricsController.class);
     private final MeterRegistry meterRegistry;
     private Counter requestCounter;
     private Timer requestTimer;
@@ -41,6 +49,8 @@ public void init() {
 
     @GetMapping("/hello")
     public String hello() {
+        logger.info("Hello endpoint called");
+        logger.warn("This is a warning log");
         // Record request count
         requestCounter.increment();
 
@@ -56,4 +66,15 @@ public String hello() {
             }
         });
     }
+
+    @GetMapping("/runtime-error")
+    public String error() {
+        logger.error("An error occurred", new RuntimeException("Test exception"));
+        return "Error logged";
+    }
+
+    @GetMapping("/error")
+    public String triggerError() {
+        throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Something went wrong!");
+    }
 }

microservices/course-composite-service/src/main/java/io/javatab/microservices/composite/course/web/CourseCompositeIntegration.java

@@ -31,11 +31,11 @@ public CourseCompositeIntegration(
     }
 
     public Mono<CourseAggregate> getCourseDetails(Long id, Jwt jwt) {
-        logger.info("JWT ===> {}", jwt.getTokenValue());
+        logger.debug("JWT ===> {}", jwt.getTokenValue());
         String courseUrl = courseServiceUrl + "/api/courses/" + id;
         String reviewUrl = reviewServiceUrl + "/api/reviews?course=" + id;
-        logger.info("Course URL ===> {}", courseUrl);
-        logger.info("Review URL ===> {}", reviewUrl);
+        logger.debug("Course URL ===> {}", courseUrl);
+        logger.debug("Review URL ===> {}", reviewUrl);
         Mono<Course> courseMono = webClient.get()
                 .uri(courseUrl)
                 .header("Authorization", "Bearer " + jwt.getTokenValue())