Skip to content

Commit 884cb52

Browse files
zeitlingerzeitlinger
committed
add example for OTLP logging via stdout and k8s
1 parent 6f8b2b0 commit 884cb52

File tree

14 files changed

+418
-0
lines changed

14 files changed

+418
-0
lines changed

‎.github/workflows/acceptance-tests.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Acceptance Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
acceptance-tests:
12+
runs-on: ubuntu-24.04
13+
steps:
14+
- name: Check out
15+
uses: actions/checkout@v4
16+
- name: Check out oats
17+
uses: actions/checkout@v4
18+
with:
19+
repository: grafana/oats
20+
ref: b4c9c7738576c10bdaf8541eede4a65ed0ee95af
21+
path: oats
22+
- name: Set up Go
23+
uses: actions/setup-go@v5
24+
with:
25+
go-version: '1.21'
26+
cache-dependency-path: oats/go.sum
27+
- name: Build Image for integration tests
28+
run: ./build-lgtm.sh
29+
- name: Run acceptance tests
30+
run: ./scripts/run-acceptance-tests.sh
31+
- name: upload log file
32+
uses: actions/upload-artifact@v4
33+
if: failure()
34+
with:
35+
name: docker-compose.log
36+
path: oats/yaml/build/**/output.log

‎logging-otlp/Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM eclipse-temurin:21-jre
2+
3+
WORKDIR /usr/src/app/
4+
5+
COPY build/libs/*SNAPSHOT.jar ./app.jar
6+
# we ignore the version (which is from upstream) and use the latest version of the grafana distribution
7+
# todo: with renovate update
8+
ADD --chmod=644 https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.10.0/opentelemetry-javaagent.jar /usr/src/app/opentelemetry-javaagent.jar
9+
ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar
10+
11+
EXPOSE 8080
12+
ENTRYPOINT [ "java", "-jar", "./app.jar" ]

‎logging-otlp/README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Exporting Application logs using JSON logging in Kubernetes
2+
3+
## Running the example
4+
5+
1. Build the Docker image using using `build.sh`
6+
2. Deploy the manifest using `kubectl apply -f k8s/` (e.g. using [k3d.sh](k3d.sh))
7+
3. Generate traffic using [generate-traffic.sh](../../../generate-traffic.sh)
8+
4. Log in to [http://localhost:3000](http://localhost:3000) with user _admin_ and password _admin_.
9+
5. Go to "Explore"
10+
6. Select "Loki" as data source

‎logging-otlp/build.gradle.kts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import org.springframework.boot.gradle.plugin.SpringBootPlugin
2+
import org.springframework.boot.gradle.tasks.bundling.BootJar
3+
4+
plugins {
5+
id("java")
6+
id("org.springframework.boot") version "3.4.0"
7+
}
8+
9+
description = "OpenTelemetry Example for Java Agent with Stdout logging"
10+
val moduleName by extra { "io.opentelemetry.examples.javagent.stdout-logging" }
11+
12+
java {
13+
toolchain {
14+
languageVersion.set(JavaLanguageVersion.of(17))
15+
}
16+
}
17+
18+
dependencies {
19+
implementation(platform(SpringBootPlugin.BOM_COORDINATES))
20+
21+
implementation("org.springframework.boot:spring-boot-starter-web")
22+
implementation("org.springframework.boot:spring-boot-starter-actuator")
23+
}
24+

‎logging-otlp/build.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" .

‎logging-otlp/k3d.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
./build.sh
6+
k3d cluster create jsonlogging || k3d cluster start jsonlogging
7+
k3d image import -c jsonlogging dice:1.1-SNAPSHOT
8+
9+
kubectl apply -f k8s/
10+
11+
kubectl wait --for=condition=ready pod -l app=dice
12+
kubectl wait --for=condition=ready --timeout=5m pod -l app=lgtm
13+
14+
kubectl port-forward service/dice 8080:8080 &
15+
kubectl port-forward service/lgtm 3000:3000 &

‎logging-otlp/k8s/collector-configmap.yaml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: otel-collector-config
5+
data:
6+
otel-collector-config.yaml: |-
7+
receivers:
8+
otlp:
9+
protocols:
10+
grpc:
11+
endpoint: 0.0.0.0:4317
12+
http:
13+
endpoint: 0.0.0.0:4318
14+
prometheus/collector: # needed if you use the docker-lgtm image
15+
config:
16+
scrape_configs:
17+
- job_name: 'opentelemetry-collector'
18+
static_configs:
19+
- targets: [ 'localhost:8888' ]
20+
filelog/otlp-json-logs:
21+
include:
22+
- /var/log/pods/*/*/*.log
23+
include_file_path: true
24+
operators:
25+
- id: container-parser
26+
type: container
27+
28+
processors:
29+
batch:
30+
resourcedetection:
31+
detectors: [ "env", "system" ]
32+
override: false
33+
34+
connectors:
35+
otlpjson:
36+
37+
exporters:
38+
otlphttp/metrics:
39+
endpoint: http://localhost:9090/api/v1/otlp
40+
otlphttp/traces:
41+
endpoint: http://localhost:4418
42+
otlphttp/logs:
43+
endpoint: http://localhost:3100/otlp
44+
debug/metrics:
45+
verbosity: detailed
46+
debug/traces:
47+
verbosity: detailed
48+
debug/logs:
49+
verbosity: detailed
50+
nop:
51+
52+
service:
53+
pipelines:
54+
traces:
55+
receivers: [ otlp ]
56+
processors: [ batch ]
57+
exporters: [ otlphttp/traces ]
58+
metrics:
59+
receivers: [ otlp, prometheus/collector ]
60+
processors: [ batch ]
61+
exporters: [ otlphttp/metrics ]
62+
logs/raw_otlpjson:
63+
receivers: [ filelog/otlp-json-logs ]
64+
# (i) no need for processors before the otlpjson connector
65+
# Declare processors in the shared "logs" pipeline below
66+
processors: [ ]
67+
exporters: [ otlpjson ]
68+
logs/otlp:
69+
receivers: [ otlp, otlpjson ]
70+
processors: [ resourcedetection, batch ]
71+
exporters: [ otlphttp/logs ]
72+
# exporters: [ otlphttp/logs, debug/logs ] # Uncomment this line to enable debug logging

‎logging-otlp/k8s/dice.yaml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: dice
5+
spec:
6+
selector:
7+
app: dice
8+
ports:
9+
- protocol: TCP
10+
port: 8080
11+
targetPort: 8080
12+
---
13+
apiVersion: apps/v1
14+
kind: Deployment
15+
metadata:
16+
name: dice
17+
spec:
18+
replicas: 1
19+
selector:
20+
matchLabels:
21+
app: dice
22+
template:
23+
metadata:
24+
labels:
25+
app: dice
26+
spec:
27+
containers:
28+
- name: dice
29+
image: dice:1.1-SNAPSHOT
30+
imagePullPolicy: Never
31+
ports:
32+
- containerPort: 8080
33+
env:
34+
- name: OTEL_EXPORTER_OTLP_ENDPOINT
35+
value: "http://lgtm:4318"
36+
- name: OTEL_LOGS_EXPORTER
37+
value: "experimental-otlp/stdout"
38+
- name: OTEL_RESOURCE_ATTRIBUTES
39+
value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging
40+
- name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_LOG_ATTRIBUTES
41+
value: "true"
42+
- name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_CAPTURE_KEY_VALUE_PAIR_ATTRIBUTES
43+
value: "true"
44+
- name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_CAPTURE_MDC_ATTRIBUTES
45+
value: "true"
46+
- name: SERVICE_NAME
47+
value: dice
48+

‎logging-otlp/k8s/lgtm.yaml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: lgtm
5+
spec:
6+
selector:
7+
app: lgtm
8+
ports:
9+
- name: grafana
10+
protocol: TCP
11+
port: 3000
12+
targetPort: 3000
13+
- name: otel-grpc
14+
protocol: TCP
15+
port: 4317
16+
targetPort: 4317
17+
- name: otel-http
18+
protocol: TCP
19+
port: 4318
20+
targetPort: 4318
21+
- name: prometheus # needed for automated tests
22+
protocol: TCP
23+
port: 9090
24+
targetPort: 9090
25+
- name: loki # needed for automated tests
26+
protocol: TCP
27+
port: 3100
28+
targetPort: 3100
29+
- name: tempo # needed for automated tests
30+
protocol: TCP
31+
port: 3200
32+
targetPort: 3200
33+
---
34+
apiVersion: apps/v1
35+
kind: Deployment
36+
metadata:
37+
name: lgtm
38+
spec:
39+
replicas: 1
40+
selector:
41+
matchLabels:
42+
app: lgtm
43+
template:
44+
metadata:
45+
labels:
46+
app: lgtm
47+
spec:
48+
containers:
49+
- name: lgtm
50+
image: grafana/otel-lgtm:latest
51+
ports:
52+
- containerPort: 3000
53+
- containerPort: 4317
54+
- containerPort: 4318
55+
- containerPort: 9090 # needed for automated tests
56+
- containerPort: 3100 # needed for automated tests
57+
- containerPort: 3200 # needed for automated tests
58+
readinessProbe:
59+
exec:
60+
command:
61+
- cat
62+
- /tmp/ready
63+
volumeMounts:
64+
- mountPath: /otel-lgtm/otelcol-config.yaml
65+
name: otel-collector-config
66+
subPath: otel-collector-config.yaml
67+
readOnly: true
68+
- mountPath: /var/log
69+
name: varlog
70+
readOnly: true
71+
- mountPath: /var/lib/docker/containers
72+
name: varlibdockercontainers
73+
readOnly: true
74+
env:
75+
- name: ENABLE_LOGS_OTELCOL
76+
value: "true"
77+
volumes:
78+
- name: otel-collector-config
79+
configMap:
80+
name: otel-collector-config
81+
- name: varlog
82+
hostPath:
83+
path: /var/log
84+
- name: varlibdockercontainers
85+
hostPath:
86+
path: /var/lib/docker/containers

‎logging-otlp/oats.yaml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats/tree/main/yaml
2+
kubernetes:
3+
dir: k8s
4+
app-service: dice
5+
app-docker-file: Dockerfile
6+
app-docker-tag: dice:1.1-SNAPSHOT
7+
app-docker-port: 8080
8+
import-images:
9+
- grafana/otel-lgtm:latest
10+
input:
11+
- path: /rolldice
12+
expected:
13+
logs:
14+
- logql: '{service_name="dice"} |~ `.*Anonymous player is rolling the dice:.*`'
15+
regexp: 'Anonymous player is rolling the dice: \d+' # uses formatted message
16+
- logql: '{service_name="dice"} |~ `.*simulating an error.*`'
17+
equals: 'Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause'
18+
attributes:
19+
deployment_environment: staging
20+
exception_message: "simulating an error"
21+
exception_type: "java.lang.RuntimeException"
22+
scope_name: "org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]"
23+
service_name: dice
24+
service_namespace: shop
25+
service_version: 1.1
26+
severity_number: 17
27+
severity_text: SEVERE
28+
exception_stacktrace: "java.lang.RuntimeException: simulating an error\n\tat com.grafana.example.RollController.index(RollController.java:21)\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source)\n\tat java.base/java.lang.reflect.Method.invoke(Unknown Source)\n\tat org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:255)\n\tat org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:188)\n\tat org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:118)\n\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:986)\n\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:891)\n\tat org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)\n\tat org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1088)\n\tat org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:978)\n\tat org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1014)\n\tat org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:903)\n\tat jakarta.servlet.http.HttpServlet.service(HttpServlet.java:564)\n\tat org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:885)\n\tat jakarta.servlet.http.HttpServlet.service(HttpServlet.java:658)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:195)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:51)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:116)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:116)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.springframework.web.servlet.v6_0.OpenTelemetryHandlerMappingFilter.doFilter(OpenTelemetryHandlerMappingFilter.java:78)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.springframework.web.filter.ServerHttpObservationFilter.doFilterInternal(ServerHttpObservationFilter.java:114)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:116)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:116)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:164)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:140)\n\tat org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:167)\n\tat org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:90)\n\tat org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:483)\n\tat org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:115)\n\tat org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:93)\n\tat org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)\n\tat org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:731)\n\tat org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:344)\n\tat org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:397)\n\tat org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:63)\n\tat org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:905)\n\tat org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1741)\n\tat org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:52)\n\tat org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1190)\n\tat org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)\n\tat org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:63)\n\tat java.base/java.lang.Thread.run(Unknown Source)\n"
29+
attribute-regexp:
30+
flags: ".*" # from loki
31+
detected_level: ".*" # from loki
32+
observed_timestamp: ".*" # from loki
33+
# thread_name: ".*" # thread name is missing when there is an exception - has nothing to do with stdout logging
34+
span_id: ".*"
35+
trace_id: ".*"
36+
container_id: ".*"
37+
host_arch: ".*"
38+
host_name: ".*"
39+
os_description: ".*"
40+
os_type: ".*"
41+
process_command_args: ".*"
42+
process_executable_path: ".*"
43+
process_pid: ".*"
44+
process_runtime_description: ".*"
45+
process_runtime_name: ".*"
46+
process_runtime_version: ".*"
47+
service_instance_id: ".*"
48+
telemetry_distro_name: ".*"
49+
telemetry_distro_version: ".*"
50+
telemetry_sdk_language: ".*"
51+
telemetry_sdk_name: ".*"
52+
telemetry_sdk_version: ".*"
53+
54+
no-extra-attributes: true
55+

0 commit comments

Comments
 (0)