From 623abc83b54ab8c907cb5b90d87352a15ac3c282 Mon Sep 17 00:00:00 2001 From: Maxime mouial Date: Tue, 26 May 2026 11:24:44 +0200 Subject: [PATCH] Switch render_config callers to schema.template Migrate the three call sites that used to invoke pkg/config/render_config (Go binary + text/template) to call tasks/schema/template.py (Python + enriched schema): - Bazel: new schema_template_config rule in pkg/config/schema/template.bzl replaces the legacy agent_config rule. Three downstream targets keep their names: //pkg/config:{agent,iot_agent,system_probe}_config. OS is picked via select() on @platforms//os. - Dev build: tasks/agent.py renames render_config -> generate_config_examples and calls generate_template in-process. The @task def generate_config is deleted; tasks/msi.py and tasks/winbuildscripts/Invoke-UnitTests.ps1 are updated to call dda inv schema.template directly. - //go:generate: dropped from cmd/cluster-agent, cmd/cluster-agent-cloudfoundry, and cmd/dogstatsd (latter file deleted entirely). The corresponding invoke tasks (cluster_agent_helpers.build_common, dogstatsd.build) now call generate_template directly. The legacy render_config binary stays in place for now. It will be removed in a follow-up PR. --- .claude/skills/create-config-field/SKILL.md | 19 +- cmd/cluster-agent-cloudfoundry/main.go | 2 - cmd/cluster-agent/main.go | 2 - cmd/dogstatsd/generate.go | 9 - cmd/dogstatsd/main_nix.go | 1 + cmd/dogstatsd/main_windows.go | 1 + docs/public/guidelines/contributing.md | 2 +- pkg/config/BUILD.bazel | 34 ++- pkg/config/schema/BUILD.bazel | 51 +++- pkg/config/schema/template.bzl | 67 +++++ .../schema/yaml/admission_controller.yaml | 13 + pkg/config/schema/yaml/core_schema.yaml | 50 +++- .../yaml/external_metrics_provider.yaml | 6 + pkg/config/schema/yaml/gpu.yaml | 26 +- pkg/config/schema/yaml/logs_config.yaml | 2 +- .../schema/yaml/system-probe_schema.yaml | 234 +++++++++--------- pkg/config/setup/common_settings.go | 2 +- pkg/config/setup/config.go | 4 +- tasks/BUILD.bazel | 10 - tasks/agent.py | 45 +--- tasks/cluster_agent_helpers.py | 21 +- tasks/dogstatsd.py | 16 +- tasks/msi.py | 6 +- tasks/schema/BUILD.bazel | 30 +++ tasks/schema/template.py | 44 ++++ .../agent_generate_config_examples_tests.py | 108 ++++++++ .../unit_tests/cluster_agent_helpers_tests.py | 69 ++++++ tasks/unit_tests/dogstatsd_tests.py | 32 +++ tasks/unit_tests/schema_template_cli_tests.py | 146 +++++++++++ tasks/winbuildscripts/Invoke-UnitTests.ps1 | 2 +- 30 files changed, 828 insertions(+), 226 deletions(-) delete mode 100644 cmd/dogstatsd/generate.go create mode 100644 pkg/config/schema/template.bzl create mode 100644 tasks/schema/BUILD.bazel create mode 100644 tasks/unit_tests/agent_generate_config_examples_tests.py create mode 100644 tasks/unit_tests/cluster_agent_helpers_tests.py create mode 100644 tasks/unit_tests/dogstatsd_tests.py create mode 100644 tasks/unit_tests/schema_template_cli_tests.py diff --git a/.claude/skills/create-config-field/SKILL.md b/.claude/skills/create-config-field/SKILL.md index 39fd0399d4ea..b78c2cf435be 100644 --- a/.claude/skills/create-config-field/SKILL.md +++ b/.claude/skills/create-config-field/SKILL.md @@ -51,7 +51,7 @@ Use `AskUserQuestion` to collect the following. If `$ARGUMENTS` provides the con 5. **Default value**: What should the default be? 6. **Description**: Human-readable description of what this field controls. 7. **Scope**: Add inline in the subsystem function, or create a dedicated setup file (`pkg/config/setup/.go`) for a group of related fields? -8. **User-facing?**: Should it be documented in `pkg/config/config_template.yaml`? +8. **User-facing?**: Should the rendered example yaml (`datadog.yaml`, `system-probe.yaml`, ...) show this setting? ### Step 2: Register the config key @@ -65,16 +65,21 @@ For a **group of related fields**, create `pkg/config/setup/.go` with a For **serverless-compatible** core agent fields, register via the `serverlessConfigComponents` slice in `config.go` instead of directly in `InitConfig`. -### Step 3: (Optional) Document in the config template +### Step 3: Regenerate the schema -If user-facing, add documentation to `pkg/config/config_template.yaml` inside the appropriate conditional block (see Template conditional column in tables above). Read existing entries nearby for the exact format (`@param`, `@env` annotations). +The example yaml configs (`datadog.yaml`, `system-probe.yaml`, etc.) are rendered from the enriched schema under `pkg/config/schema/yaml/`. The schema is derived from the running agent, so build first, then regenerate: + +```bash +dda inv agent.build --build-exclude=systemd +dda inv schema.generate --agent-bin=./bin/agent/agent +``` + +Commit the resulting changes under `pkg/config/schema/yaml/` alongside your Go code. ### Step 4: Verify -1. Build: `dda inv agent.build --build-exclude=systemd` (or `dda inv system-probe.build`) -2. Lint: `dda inv linter.go` -3. If the template was modified: `dda inv generate-config` -4. Report the results to the user. +1. Lint: `dda inv linter.go` +2. Report the results to the user. ## Key Methods Reference diff --git a/cmd/cluster-agent-cloudfoundry/main.go b/cmd/cluster-agent-cloudfoundry/main.go index d0118440e1eb..6680f63fbec7 100644 --- a/cmd/cluster-agent-cloudfoundry/main.go +++ b/cmd/cluster-agent-cloudfoundry/main.go @@ -5,8 +5,6 @@ //go:build !windows && clusterchecks -//go:generate go run ../../pkg/config/render_config/render_config.go dcacf ../../pkg/config/config_template.yaml ../../cloudfoundry.yaml - //nolint:revive // TODO(PLINT) Fix revive linter package main diff --git a/cmd/cluster-agent/main.go b/cmd/cluster-agent/main.go index 41e31dcc1a14..8d8806269760 100644 --- a/cmd/cluster-agent/main.go +++ b/cmd/cluster-agent/main.go @@ -5,8 +5,6 @@ //go:build !windows && kubeapiserver -//go:generate go run ../../pkg/config/render_config/render_config.go dca ../../pkg/config/config_template.yaml ../../Dockerfiles/cluster-agent/datadog-cluster.yaml - package main import ( diff --git a/cmd/dogstatsd/generate.go b/cmd/dogstatsd/generate.go deleted file mode 100644 index 0c20f54f6751..000000000000 --- a/cmd/dogstatsd/generate.go +++ /dev/null @@ -1,9 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:generate go run ../../pkg/config/render_config/render_config.go dogstatsd ../../pkg/config/config_template.yaml ./dist/dogstatsd.yaml - -//nolint:revive // TODO(AML) Fix revive linter -package main diff --git a/cmd/dogstatsd/main_nix.go b/cmd/dogstatsd/main_nix.go index cce9de52cbd8..4d5ab5df1fbe 100644 --- a/cmd/dogstatsd/main_nix.go +++ b/cmd/dogstatsd/main_nix.go @@ -5,6 +5,7 @@ //go:build !windows +// Main package for the dogstatsd binary package main import ( diff --git a/cmd/dogstatsd/main_windows.go b/cmd/dogstatsd/main_windows.go index ed5529756fca..cfbcd54fc727 100644 --- a/cmd/dogstatsd/main_windows.go +++ b/cmd/dogstatsd/main_windows.go @@ -3,6 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +// Main package for the dogstatsd binary package main import ( diff --git a/docs/public/guidelines/contributing.md b/docs/public/guidelines/contributing.md index 360ba86c02a1..d459cfaa94f7 100644 --- a/docs/public/guidelines/contributing.md +++ b/docs/public/guidelines/contributing.md @@ -33,7 +33,7 @@ In order to ease/speed up our review, here are some items you can check/improve - [X] The added code comes with tests. - [X] The CI is green, all tests are passing (required or not). - [X] All applicable labels are set on the PR (see [PR labels list](#pr-labels)). -- [X] If applicable, the [config template](https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config_template.yaml) has been updated. +- [X] If applicable, the [config schema](https://github.com/DataDog/datadog-agent/tree/main/pkg/config/schema/yaml) has been regenerated (`dda inv schema.generate --agent-bin=./bin/agent/agent`). /// /// note diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index 7affd601a33f..2b0513bf783d 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -1,29 +1,45 @@ """Agent configuration files.""" -load("//pkg/config/render_config:agent_config.bzl", "agent_config") +load("//pkg/config/schema:template.bzl", "schema_template_config") package(default_visibility = ["//visibility:public"]) -agent_config( +# Target OS for the rendered config. The legacy `render_config` Go binary +# used `runtime.GOOS` of the build host; the schema-template equivalent +# is a Bazel `select()` on `@platforms//os` so cross-platform builds +# (e.g. windows packages on a linux host) produce the right per-OS +# defaults. +_OS_TARGET = select({ + "@platforms//os:linux": "linux", + "@platforms//os:windows": "windows", + "@platforms//os:macos": "darwin", +}) + +schema_template_config( name = "agent_config", + srcs = ["//pkg/config/schema:core_schema_subfiles"], out = "datadog.yaml", build_type = "agent-py3", - template = "config_template.yaml", + os_target = _OS_TARGET, + top_schema = "//pkg/config/schema:yaml/core_schema.yaml", ) -agent_config( +schema_template_config( name = "iot_agent_config", + srcs = ["//pkg/config/schema:core_schema_subfiles"], out = "iot-agent.yaml", build_type = "iot-agent", - template = "config_template.yaml", + os_target = _OS_TARGET, + top_schema = "//pkg/config/schema:yaml/core_schema.yaml", ) -agent_config( +schema_template_config( name = "system_probe_config", out = "system-probe.yaml", build_type = "system-probe", - template = "system-probe_template.yaml", + os_target = _OS_TARGET, + top_schema = "//pkg/config/schema:yaml/system-probe_schema.yaml", ) -# TODO: Add targets for each package, as needed: dogstatsd, dca, dcacf -# That would get easier if we auto-pick the template based on the build type. +# TODO: Add targets for each package, as needed: dogstatsd, dca, dcacf. +# That would get easier if we auto-pick the schema based on the build type. diff --git a/pkg/config/schema/BUILD.bazel b/pkg/config/schema/BUILD.bazel index 38903c8cf3b6..772ae8b4d52f 100644 --- a/pkg/config/schema/BUILD.bazel +++ b/pkg/config/schema/BUILD.bazel @@ -1,8 +1,50 @@ +load("@bazel_lib//lib:run_binary.bzl", "run_binary") load("@rules_go//go:def.bzl", "go_library", "go_test") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_files") load("//bazel/rules:zstd.bzl", "zstd_compress") +# Top schemas exposed as labels for downstream rules (e.g. the +# `schema_template_config` rule in :template.bzl). +exports_files( + [ + "yaml/core_schema.yaml", + "yaml/system-probe_schema.yaml", + ], + visibility = ["//visibility:public"], +) + +# Per-section sub-files referenced by yaml/core_schema.yaml via $ref. +# Listed explicitly rather than globbed (per QUESTIONS-2 Q12) so adding +# or removing a sub-file is a visible diff. +filegroup( + name = "core_schema_subfiles", + srcs = [ + "yaml/admission_controller.yaml", + "yaml/agent_telemetry.yaml", + "yaml/apm_config.yaml", + "yaml/cluster_agent.yaml", + "yaml/cluster_checks.yaml", + "yaml/compliance_config.yaml", + "yaml/container_image.yaml", + "yaml/container_lifecycle.yaml", + "yaml/external_metrics_provider.yaml", + "yaml/gpu.yaml", + "yaml/internal_profiling.yaml", + "yaml/logs_config.yaml", + "yaml/multi_region_failover.yaml", + "yaml/orchestrator_explorer.yaml", + "yaml/otelcollector.yaml", + "yaml/private_action_runner.yaml", + "yaml/process_config.yaml", + "yaml/remote_configuration.yaml", + "yaml/runtime_security_config.yaml", + "yaml/sbom.yaml", + "yaml/snmp_listener.yaml", + ], + visibility = ["//visibility:public"], +) + ZSTD_ARGS = [ "--no-check", # match DataDog/zstd Go library behavior: no XXH64 frame checksum "-5", # match DataDog/zstd: DefaultCompression = 5 @@ -12,12 +54,15 @@ ZSTD_ARGS = [ # yaml/apm_config.yaml, ...) via $ref. Merge them into a single file before # compression so the embedded Go artifact stays a single self-contained # schema and the Go loader doesn't need to handle $ref resolution. -genrule( +run_binary( name = "merged_core_schema", srcs = glob(["yaml/*.yaml"]), outs = ["core_schema.merged.yaml"], - cmd = "$(execpath //tasks:merge_schema) $(execpath yaml/core_schema.yaml) $@", - tools = ["//tasks:merge_schema"], + args = [ + "$(execpath yaml/core_schema.yaml)", + "$@", + ], + tool = "//tasks/schema:merge_schema", ) zstd_compress( diff --git a/pkg/config/schema/template.bzl b/pkg/config/schema/template.bzl new file mode 100644 index 000000000000..e6a0e96fb22c --- /dev/null +++ b/pkg/config/schema/template.bzl @@ -0,0 +1,67 @@ +"""Bazel rule for rendering a config example YAML from an enriched schema. + +Inputs (per rule invocation): + - top_schema: the top-level schema file (core_schema.yaml or + system-probe_schema.yaml). For the core agent flavors this entry + references per-section sub-files via $ref, so those sub-files must + also be supplied via `srcs`. + - srcs: additional schema sub-files that the top_schema references. + - build_type: agent-py3, iot-agent, dogstatsd, dca, dcacf, or + system-probe. + - os_target: linux, windows, or darwin. + - out: the rendered example yaml. +""" + +def _schema_template_config_impl(ctx): + args = ctx.actions.args() + args.add(ctx.file.top_schema.path) + args.add(ctx.attr.build_type) + args.add(ctx.attr.os_target) + args.add(ctx.outputs.out.path) + + ctx.actions.run( + mnemonic = "SchemaTemplateConfig", + executable = ctx.executable._template, + arguments = [args], + inputs = [ctx.file.top_schema] + ctx.files.srcs, + outputs = [ctx.outputs.out], + progress_message = "Rendering %s (%s, %s)" % ( + ctx.outputs.out.short_path, + ctx.attr.build_type, + ctx.attr.os_target, + ), + ) + return DefaultInfo(files = depset([ctx.outputs.out])) + +schema_template_config = rule( + implementation = _schema_template_config_impl, + attrs = { + "build_type": attr.string( + doc = "One of agent-py3, iot-agent, dogstatsd, dca, dcacf, system-probe.", + mandatory = True, + ), + "os_target": attr.string( + doc = "Target OS: one of linux, windows, darwin. Typically populated via `select()` on `@platforms//os`.", + mandatory = True, + ), + "top_schema": attr.label( + doc = "The top-level enriched schema YAML file.", + allow_single_file = [".yaml"], + mandatory = True, + ), + "srcs": attr.label_list( + doc = "Additional schema sub-files referenced by the top schema via $ref. Empty for system-probe (single-file schema).", + allow_files = [".yaml"], + default = [], + ), + "out": attr.output( + doc = "Path of the rendered example yaml file.", + mandatory = True, + ), + "_template": attr.label( + default = Label("//tasks/schema:schema_template"), + executable = True, + cfg = "exec", + ), + }, +) diff --git a/pkg/config/schema/yaml/admission_controller.yaml b/pkg/config/schema/yaml/admission_controller.yaml index 85008d1c8375..3cdeb334c283 100644 --- a/pkg/config/schema/yaml/admission_controller.yaml +++ b/pkg/config/schema/yaml/admission_controller.yaml @@ -618,6 +618,19 @@ properties: type: string default: datadog/ingress-nginx-injection comment: ingress-nginx injection configuration + init_run_as_group: + node_type: setting + type: integer + default: 82 + init_run_as_user: + node_type: setting + type: integer + default: 101 + comment: |- + Non-root UID/GID for the injected init container. Defaults match the + stock datadog/ingress-nginx-injection image, which declares no USER and + would otherwise be rejected under runAsNonRoot. Set to a negative value + to leave the security context unset and honor a custom init_image's own USER. module_mount_path: node_type: setting type: string diff --git a/pkg/config/schema/yaml/core_schema.yaml b/pkg/config/schema/yaml/core_schema.yaml index a0fbd19616b2..4b6f30797388 100644 --- a/pkg/config/schema/yaml/core_schema.yaml +++ b/pkg/config/schema/yaml/core_schema.yaml @@ -810,7 +810,6 @@ properties: default: true visibility: public description: Collect GPU related host tags - example: 'false' tags: - template_section:Common - full-agent-only:true @@ -1111,6 +1110,44 @@ properties: example: '"http://127.0.0.1:8080"' tags: - template_section:Common + dual_ship: + node_type: setting + type: boolean + default: false + visibility: public + description: |- + When false (default), the Observability Pipelines Worker replaces the primary + Datadog logs intake as the sole destination. When true, logs are sent to BOTH + the primary Datadog intake and the Observability Pipelines Worker, which is + useful for evaluating OPW without interrupting the existing flow of telemetry + to Datadog. + tags: + - template_section:Common + comment: "dual_ship is logs-only: there is no equivalent dual-shipping\ + \ code path for metrics, so\nthese keys live outside bindVectorOptions\ + \ to avoid registering an unused metrics variant.\n\ndual_ship: when\ + \ false (default), OPW replaces the primary Datadog endpoint and is\ + \ the only\ndestination logs are shipped to. When true, Datadog remains\ + \ the primary endpoint and OPW is\nadded as an additional endpoint \u2014\ + \ intended for operators evaluating OPW without interrupting\nthe existing\ + \ flow of telemetry to Datadog.\n\ndual_ship_reliable: when dual_ship=true,\ + \ controls whether the OPW additional endpoint applies\nbackpressure\ + \ to the main pipeline on failure (true) or is best-effort (false, the\ + \ default).\nBest-effort is the safer default: an unreachable OPW must\ + \ not block delivery to Datadog." + dual_ship_reliable: + node_type: setting + type: boolean + default: false + visibility: public + description: |- + Only meaningful when `dual_ship` is true. When false (default), the OPW + destination is treated as best-effort: failures to OPW do not apply + backpressure to the main pipeline, so an unhealthy OPW cannot stall delivery + to Datadog. Set to true to make the OPW destination reliable, meaning its + failures participate in flow control just like the primary Datadog endpoint. + tags: + - template_section:Common traces: node_type: section type: object @@ -9916,6 +9953,17 @@ properties: node_type: section type: object properties: + dual_ship: + node_type: setting + type: boolean + default: false + comment: "Legacy vector.* aliases for dual_ship keys \u2014 users still\ + \ on the legacy prefix must not have\ndual_ship=true silently dropped\ + \ when the fallback in obsPipelineWorkerDualShip reads these keys." + dual_ship_reliable: + node_type: setting + type: boolean + default: false enabled: node_type: setting type: boolean diff --git a/pkg/config/schema/yaml/external_metrics_provider.yaml b/pkg/config/schema/yaml/external_metrics_provider.yaml index 3f3742b26965..77849b93815d 100644 --- a/pkg/config/schema/yaml/external_metrics_provider.yaml +++ b/pkg/config/schema/yaml/external_metrics_provider.yaml @@ -13,6 +13,12 @@ properties: type: string default: '' comment: Override the Datadog APP Key for external metrics endpoint + autoscaler_autogen_label_selector: + node_type: setting + type: string + default: '' + comment: Label selector to filter which HPAs and WPAs the DCA generates DatadogMetrics + for (e.g. "app.kubernetes.io/managed-by!=keda-operator") batch_window: node_type: setting type: integer diff --git a/pkg/config/schema/yaml/gpu.yaml b/pkg/config/schema/yaml/gpu.yaml index 5f4a9cdac75c..896917cf1f1b 100644 --- a/pkg/config/schema/yaml/gpu.yaml +++ b/pkg/config/schema/yaml/gpu.yaml @@ -3,11 +3,24 @@ $id: https://raw.githubusercontent.com/DataDog/schema/main/agent/gpu.yaml.schema node_type: section type: object visibility: public -description: GPU monitoring configuration. +description: |- + Configuration for GPU Monitoring on non-containerized Linux hosts. + Both this section and the gpu_monitoring section in system-probe.yaml must be configured. tags: - template_section:Common - full-agent-only:true properties: + enabled: + node_type: setting + type: boolean + default: false + visibility: public + description: |- + Set to true to enable the GPU check in the core Agent. + Required for GPU monitoring on non-containerized Linux hosts. + Must be used together with gpu_monitoring.enabled: true in system-probe.yaml. + tags: + - template_section:Common nccl: node_type: section type: object @@ -60,17 +73,6 @@ properties: default: [] items: type: string - enabled: - node_type: setting - type: boolean - default: false - visibility: public - description: |- - Set to true to enable the GPU check in the core Agent. - Required for GPU monitoring on non-containerized Linux hosts. - Must be used together with gpu_monitoring.enabled: true in system-probe.yaml. - tags: - - template_section:Common integrate_with_workloadmeta_processes: node_type: setting type: boolean diff --git a/pkg/config/schema/yaml/logs_config.yaml b/pkg/config/schema/yaml/logs_config.yaml index f767a7fffe1f..3689d6206b31 100644 --- a/pkg/config/schema/yaml/logs_config.yaml +++ b/pkg/config/schema/yaml/logs_config.yaml @@ -220,7 +220,7 @@ properties: Controls how wildcard file log source are prioritized when there are more files that match wildcard log configurations than the `logs_config.open_files_limit` - Choices are 'by_name' and 'by_modification_time'. See config_template.yaml for full details. + Choices are 'by_name' and 'by_modification_time'. See pkg/config/schema/yaml/ for full details. WARNING: 'by_modification_time' is less performant than 'by_name' and will trigger more disk I/O at the wildcard log paths diff --git a/pkg/config/schema/yaml/system-probe_schema.yaml b/pkg/config/schema/yaml/system-probe_schema.yaml index cc3544dad914..5e0205ba9184 100644 --- a/pkg/config/schema/yaml/system-probe_schema.yaml +++ b/pkg/config/schema/yaml/system-probe_schema.yaml @@ -974,10 +974,8 @@ properties: default: 16 max_concurrent_requests: node_type: setting - type: number + type: integer default: 0 - tags: - - golang_type:int max_http_stats_buffered: node_type: setting type: integer @@ -2494,6 +2492,125 @@ properties: - DD_EVENT_MONITORING_CONFIG_SYSCALLS_MONITOR_ENABLED - DD_RUNTIME_SECURITY_CONFIG_SYSCALLS_MONITOR_ENABLED tags: [] + gpu_monitoring: + node_type: section + title: System Probe GPU Monitoring module + type: object + visibility: public + description: |- + Configuration for the GPU Monitoring module, which uses eBPF probes to collect + per-process GPU metrics on Linux hosts (kernel 5.8+). + Required for GPU monitoring on non-containerized Linux hosts alongside + gpu.enabled: true in datadog.yaml. + tags: [] + properties: + enabled: + node_type: setting + type: boolean + default: false + visibility: public + description: |- + Set to true to enable the GPU Monitoring module. This loads the eBPF programs + that collect per-process GPU utilization, memory, and performance metrics. + Must be used together with gpu.enabled: true in datadog.yaml. + tags: [] + attacher_detailed_logs: + node_type: setting + type: boolean + default: false + cgroup_reapply_infinitely: + node_type: setting + type: boolean + default: false + cgroup_reapply_interval: + node_type: setting + type: number + default: 30s + format: duration + tags: + - golang_type:duration + configure_cgroup_perms: + node_type: setting + type: boolean + default: false + device_cache_refresh_interval: + node_type: setting + type: number + default: 5s + format: duration + tags: + - golang_type:duration + enable_ebpf_probes: + node_type: setting + type: boolean + default: true + enable_fatbin_parsing: + node_type: setting + type: boolean + default: false + fatbin_request_queue_size: + node_type: setting + type: integer + default: 100 + initial_process_sync: + node_type: setting + type: boolean + default: true + nvml_lib_path: + node_type: setting + type: string + default: '' + prm_endpoint_enabled: + node_type: setting + type: boolean + default: true + process_scan_interval_seconds: + node_type: setting + type: integer + default: 5 + ring_buffer_pages_per_device: + node_type: setting + type: integer + default: 32 + ringbuffer_flush_interval: + node_type: setting + type: number + default: 1s + format: duration + tags: + - golang_type:duration + ringbuffer_wakeup_size: + node_type: setting + type: integer + default: 3000 + streams: + node_type: section + type: object + properties: + max_active: + node_type: setting + type: integer + default: 100 + max_kernel_launches: + node_type: setting + type: integer + default: 1000 + max_mem_alloc_events: + node_type: setting + type: integer + default: 1000 + max_pending_kernel_spans: + node_type: setting + type: integer + default: 1000 + max_pending_memory_spans: + node_type: setting + type: integer + default: 1000 + timeout_seconds: + node_type: setting + type: integer + default: 30 windows_crash_detection: node_type: section title: Datadog Agent Windows Crash Detection module @@ -2746,117 +2863,6 @@ properties: node_type: setting type: boolean default: false - gpu_monitoring: - node_type: section - type: object - title: System Probe GPU Monitoring module - description: |- - Configuration for the GPU Monitoring module, which uses eBPF probes to collect - per-process GPU metrics on Linux hosts (kernel 5.8+). - Required for GPU monitoring on non-containerized Linux hosts alongside - gpu.enabled: true in datadog.yaml. - properties: - attacher_detailed_logs: - node_type: setting - type: boolean - default: false - cgroup_reapply_infinitely: - node_type: setting - type: boolean - default: false - cgroup_reapply_interval: - node_type: setting - type: number - default: 30s - format: duration - tags: - - golang_type:duration - configure_cgroup_perms: - node_type: setting - type: boolean - default: false - device_cache_refresh_interval: - node_type: setting - type: number - default: 5s - format: duration - tags: - - golang_type:duration - enable_ebpf_probes: - node_type: setting - type: boolean - default: true - enable_fatbin_parsing: - node_type: setting - type: boolean - default: false - enabled: - node_type: setting - type: boolean - default: false - fatbin_request_queue_size: - node_type: setting - type: integer - default: 100 - initial_process_sync: - node_type: setting - type: boolean - default: true - nvml_lib_path: - node_type: setting - type: string - default: '' - prm_endpoint_enabled: - node_type: setting - type: boolean - default: true - process_scan_interval_seconds: - node_type: setting - type: integer - default: 5 - ring_buffer_pages_per_device: - node_type: setting - type: integer - default: 32 - ringbuffer_flush_interval: - node_type: setting - type: number - default: 1s - format: duration - tags: - - golang_type:duration - ringbuffer_wakeup_size: - node_type: setting - type: integer - default: 3000 - streams: - node_type: section - type: object - properties: - max_active: - node_type: setting - type: integer - default: 100 - max_kernel_launches: - node_type: setting - type: integer - default: 1000 - max_mem_alloc_events: - node_type: setting - type: integer - default: 1000 - max_pending_kernel_spans: - node_type: setting - type: integer - default: 1000 - max_pending_memory_spans: - node_type: setting - type: integer - default: 1000 - timeout_seconds: - node_type: setting - type: integer - default: 30 ignore_host_etc: node_type: setting type: boolean diff --git a/pkg/config/setup/common_settings.go b/pkg/config/setup/common_settings.go index 809423a5be9a..5d6601766f18 100644 --- a/pkg/config/setup/common_settings.go +++ b/pkg/config/setup/common_settings.go @@ -1909,7 +1909,7 @@ func logsagent(config pkgconfigmodel.Setup) { // Controls how wildcard file log source are prioritized when there are more files // that match wildcard log configurations than the `logs_config.open_files_limit` // - // Choices are 'by_name' and 'by_modification_time'. See config_template.yaml for full details. + // Choices are 'by_name' and 'by_modification_time'. See pkg/config/schema/yaml/ for full details. // // WARNING: 'by_modification_time' is less performant than 'by_name' and will trigger // more disk I/O at the wildcard log paths diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 0cba64ecb2ec..79afeead39df 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -949,8 +949,8 @@ func setupFipsEndpoints(config pkgconfigmodel.Config) error { config.Set("skip_ssl_validation", !config.GetBool("fips.tls_verify"), pkgconfigmodel.SourceAgentRuntime) } - // The following overwrites should be sync with the documentation for the fips.enabled config setting in the - // config_template.yaml + // The following overwrites should be kept in sync with the documentation for the fips.enabled config + // setting in pkg/config/schema/yaml/. // Metrics config.Set("dd_url", protocol+urlFor(metrics), pkgconfigmodel.SourceAgentRuntime) diff --git a/tasks/BUILD.bazel b/tasks/BUILD.bazel index 836bd626d16e..5e74afb0afff 100644 --- a/tasks/BUILD.bazel +++ b/tasks/BUILD.bazel @@ -15,16 +15,6 @@ py_library( exports_files(["core_checks.py"]) -# Used by //pkg/config/schema:merged_core_schema to inline the per-section -# sub-files referenced by core_schema.yaml's $refs. -py_binary( - name = "merge_schema", - srcs = ["schema/merge_schema.py"], - main = "schema/merge_schema.py", - visibility = ["//pkg/config/schema:__pkg__"], - deps = ["@py_dev_requirements//pyyaml"], -) - # No `data = [...]` for renovate.json / deps/ / .renovate-untracked.json on # purpose: the script resolves them via $BUILD_WORKSPACE_DIRECTORY (set by # `bazel run`) so it reads the live workspace, not the bazel sandbox. Adding diff --git a/tasks/agent.py b/tasks/agent.py index c660b079c4e4..942d090de6be 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -40,6 +40,7 @@ from tasks.rtloader import install_with_bazel as rtloader_install_with_bazel from tasks.rtloader import make as rtloader_make from tasks.schema.generate import compress as schema_compress +from tasks.schema.template import CORE_SCHEMA_FILE, SYSPROBE_SCHEMA_FILE, generate_template from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars # constants @@ -170,9 +171,8 @@ def build( ) with gitlab_section("Generate configuration files", collapsed=True): - render_config( + generate_config_examples( ctx, - env=env, flavor=flavor, skip_assets=skip_assets, build_tags=build_tags, @@ -181,20 +181,21 @@ def build( ) -def render_config(ctx, env, flavor, skip_assets, build_tags, development, windows_sysprobe): - # Remove cross-compiling bits to render config - env.update({"GOOS": "", "GOARCH": ""}) +_PLATFORM_TO_OS_TARGET = { + "linux": "linux", + "win32": "windows", + "darwin": "darwin", +} - # Render the Agent configuration file template - build_type = "agent-py3" - if flavor.is_iot(): - build_type = "iot-agent" - generate_config(ctx, build_type=build_type, output_file="./cmd/agent/dist/datadog.yaml", env=env) +def generate_config_examples(ctx, flavor, skip_assets, build_tags, development, windows_sysprobe): + os_target = _PLATFORM_TO_OS_TARGET[sys.platform] + + build_type = "iot-agent" if flavor.is_iot() else "agent-py3" + generate_template(CORE_SCHEMA_FILE, "./cmd/agent/dist/datadog.yaml", build_type, os_target) - # On Linux and MacOS, render the system-probe configuration file template if sys.platform != 'win32' or windows_sysprobe: - generate_config(ctx, build_type="system-probe", output_file="./cmd/agent/dist/system-probe.yaml", env=env) + generate_template(SYSPROBE_SCHEMA_FILE, "./cmd/agent/dist/system-probe.yaml", "system-probe", os_target) if not skip_assets: refresh_assets(ctx, build_tags, development=development, flavor=flavor.name, windows_sysprobe=windows_sysprobe) @@ -898,26 +899,6 @@ def upload_integration_to_cache(ctx, python, bucket, branch, integrations_dir, b ctx.run(f"{AWS_CMD} s3 cp {wheel_path} s3://{bucket}/{target_name} --acl public-read") -@task() -def generate_config(ctx, build_type, output_file, env=None): - """ - Generates the datadog.yaml configuration file. - """ - args = { - "go_file": "./pkg/config/render_config/render_config.go", - "build_type": build_type, - "template_file": "./pkg/config/config_template.yaml", - "output_file": output_file, - } - if build_type == "system-probe": - args["template_file"] = "./pkg/config/system-probe_template.yaml" - elif build_type == "security-agent": - args["template_file"] = "./pkg/config/security-agent_template.yaml" - - cmd = "go run {go_file} {build_type} {template_file} {output_file}" - return ctx.run(cmd.format(**args), env=env or {}) - - @task() def build_remote_agent(ctx, env=None): """ diff --git a/tasks/cluster_agent_helpers.py b/tasks/cluster_agent_helpers.py index 4c73844ab4c5..3701a5c3fbc1 100644 --- a/tasks/cluster_agent_helpers.py +++ b/tasks/cluster_agent_helpers.py @@ -8,6 +8,14 @@ from tasks.libs.common.go import go_build from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags, get_version from tasks.schema.generate import compress as schema_compress +from tasks.schema.template import CORE_SCHEMA_FILE, generate_template + +# Maps cluster-agent binary suffix to (build_type, output file). +# Empty suffix -> mainline cluster-agent (dca); -cloudfoundry -> dcacf. +_CLUSTER_AGENT_RENDER_TARGETS = { + "": ("dca", "./Dockerfiles/cluster-agent/datadog-cluster.yaml"), + "-cloudfoundry": ("dcacf", "./cloudfoundry.yaml"), +} def build_common( @@ -48,14 +56,11 @@ def build_common( coverage=cover, ) - # Render the configuration file template - # - # We need to remove cross compiling bits if any because go generate must - # build and execute in the native platform - env.update({"GOOS": "", "GOARCH": ""}) - - cmd = "go generate -mod={go_mod} -tags '{build_tags}' {repo_path}/cmd/cluster-agent{suffix}" - ctx.run(cmd.format(go_mod=go_mod, build_tags=" ".join(build_tags), repo_path=REPO_PATH, suffix=bin_suffix), env=env) + # Render the configuration file template. The cluster-agent and the + # cloudfoundry variant only ship on linux, so we always target linux + # (matches the legacy `go generate` behavior on the native build host). + build_type, output = _CLUSTER_AGENT_RENDER_TARGETS[bin_suffix] + generate_template(CORE_SCHEMA_FILE, output, build_type, "linux") if not skip_assets: refresh_assets_common( diff --git a/tasks/dogstatsd.py b/tasks/dogstatsd.py index eb71a388aea0..9afeedeecea0 100644 --- a/tasks/dogstatsd.py +++ b/tasks/dogstatsd.py @@ -15,6 +15,7 @@ from tasks.flavor import AgentFlavor from tasks.libs.common.go import go_build from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags +from tasks.schema.template import CORE_SCHEMA_FILE, generate_template from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars # constants @@ -22,6 +23,7 @@ STATIC_BIN_PATH = os.path.join(".", "bin", "static") MAX_BINARY_SIZE = 44 * 1024 DOGSTATSD_TAG = "datadog/dogstatsd:master" +DOGSTATSD_CONFIG_OUTPUT = "./cmd/dogstatsd/dist/dogstatsd.yaml" @task @@ -71,16 +73,10 @@ def build( check_deadcode=os.getenv("DEPLOY_AGENT") == "true", ) - # Render the configuration file template - # - # We need to remove cross compiling bits if any because go generate must - # build and execute in the native platform - env = { - "GOOS": "", - "GOARCH": "", - } - cmd = "go generate -mod={} {}/cmd/dogstatsd" - ctx.run(cmd.format(go_mod, REPO_PATH), env=env) + # Render the configuration file template. The dogstatsd binary ships + # on linux containers, so we always target linux (matches the legacy + # `go generate` behavior on the native build host). + generate_template(CORE_SCHEMA_FILE, DOGSTATSD_CONFIG_OUTPUT, "dogstatsd", "linux") if static and sys.platform.startswith("linux"): cmd = "file {bin_name} " diff --git a/tasks/msi.py b/tasks/msi.py index 385c11d6c08b..e9d7e923f12c 100644 --- a/tasks/msi.py +++ b/tasks/msi.py @@ -502,7 +502,11 @@ def test(ctx, vstudio_root=None, arch="x64", debug=False): # Generate the config file if not ctx.run( - f'dda inv -- -e agent.generate-config --build-type="agent-py3" --output-file="{build_outdir}\\datadog.yaml"', + 'dda inv -- -e schema.template ' + '--schema=./pkg/config/schema/yaml/core_schema.yaml ' + '--build-type=agent-py3 ' + '--os-target=windows ' + f'--output="{build_outdir}\\datadog.yaml"', warn=True, env=env, ): diff --git a/tasks/schema/BUILD.bazel b/tasks/schema/BUILD.bazel new file mode 100644 index 000000000000..15fda119fa9c --- /dev/null +++ b/tasks/schema/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +package(default_visibility = ["//visibility:private"]) + +# Used by //pkg/config/schema:merged_core_schema to inline the per-section +# sub-files referenced by core_schema.yaml's $refs. +py_binary( + name = "merge_schema", + srcs = ["merge_schema.py"], + visibility = ["//pkg/config/schema:__pkg__"], + deps = ["@py_dev_requirements//pyyaml"], +) + +# Used by //pkg/config/schema:template.bzl (schema_template_config) to +# render a config example yaml from an enriched schema. Wraps the +# `generate_template` function from template.py so Bazel actions can +# invoke it without going through the full invoke task collection. +py_binary( + name = "schema_template", + srcs = [ + "merge_schema.py", + "template.py", + ], + main = "template.py", + visibility = ["//visibility:public"], + deps = [ + "@py_dev_requirements//invoke", + "@py_dev_requirements//pyyaml", + ], +) diff --git a/tasks/schema/template.py b/tasks/schema/template.py index 88f7ebf6ee6c..3b0c3062fe5f 100644 --- a/tasks/schema/template.py +++ b/tasks/schema/template.py @@ -3,6 +3,7 @@ """ import os +import sys import textwrap from invoke import task @@ -148,6 +149,12 @@ def _env_type_for_json(node): VALID_BUILD_TYPES = list(build_type_to_section.keys()) VALID_OS_TARGETS = list(default_path.keys()) +# Canonical paths to the enriched schema files. Callers (tasks/agent.py, +# tasks/cluster_agent_helpers.py, tasks/dogstatsd.py) import these so the +# schema location is defined in exactly one place. +CORE_SCHEMA_FILE = "./pkg/config/schema/yaml/core_schema.yaml" +SYSPROBE_SCHEMA_FILE = "./pkg/config/schema/yaml/system-probe_schema.yaml" + # build_types that use the core schema vs the system-probe schema _SYSPROBE_BUILD_TYPES = {"system-probe"} @@ -454,3 +461,40 @@ def template_all(ctx, core_schema, sysprobe_schema, output_dir): dest = os.path.join(output_dir, f"{build_type}_{os_target}.yaml") generate_template(schema, dest, build_type, os_target) print(f" {dest}") + + +def main(argv): + """CLI entry point for `bazel run //tasks/schema:schema_template` and direct + `python -m` invocation. Mirrors the `template` invoke task, minus the + Context dependency, so it can be called from a Bazel py_binary + without going through the full invoke task collection.""" + if len(argv) != 5: + print( + f"usage: {argv[0]} ", + file=sys.stderr, + ) + return 2 + _, schema, build_type, os_target, output = argv + + if build_type not in VALID_BUILD_TYPES: + print( + f"Invalid build_type '{build_type}'. Must be one of: {', '.join(VALID_BUILD_TYPES)}", + file=sys.stderr, + ) + return 1 + if os_target not in VALID_OS_TARGETS: + print( + f"Invalid os_target '{os_target}'. Must be one of: {', '.join(VALID_OS_TARGETS)}", + file=sys.stderr, + ) + return 1 + if not os.path.isfile(schema): + print(f"Schema file not found: {schema}", file=sys.stderr) + return 1 + + generate_template(schema, output, build_type, os_target) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tasks/unit_tests/agent_generate_config_examples_tests.py b/tasks/unit_tests/agent_generate_config_examples_tests.py new file mode 100644 index 000000000000..34b5f16a41f5 --- /dev/null +++ b/tasks/unit_tests/agent_generate_config_examples_tests.py @@ -0,0 +1,108 @@ +import unittest +from unittest.mock import MagicMock, patch + +from tasks.agent import generate_config_examples +from tasks.flavor import AgentFlavor +from tasks.schema.template import CORE_SCHEMA_FILE, SYSPROBE_SCHEMA_FILE + + +class TestGenerateConfigExamples(unittest.TestCase): + @patch("tasks.agent.refresh_assets") + @patch("tasks.agent.generate_template") + @patch("tasks.agent.sys") + def test_linux_base_flavor_renders_agent_and_sysprobe(self, sys_mod, gen, refresh): + sys_mod.platform = "linux" + ctx = MagicMock() + + generate_config_examples( + ctx, + flavor=AgentFlavor.base, + skip_assets=True, + build_tags=["python"], + development=False, + windows_sysprobe=False, + ) + + gen.assert_any_call(CORE_SCHEMA_FILE, "./cmd/agent/dist/datadog.yaml", "agent-py3", "linux") + gen.assert_any_call(SYSPROBE_SCHEMA_FILE, "./cmd/agent/dist/system-probe.yaml", "system-probe", "linux") + self.assertEqual(gen.call_count, 2) + refresh.assert_not_called() + + @patch("tasks.agent.refresh_assets") + @patch("tasks.agent.generate_template") + @patch("tasks.agent.sys") + def test_iot_flavor_uses_iot_agent_build_type(self, sys_mod, gen, _refresh): + sys_mod.platform = "linux" + ctx = MagicMock() + + generate_config_examples( + ctx, + flavor=AgentFlavor.iot, + skip_assets=True, + build_tags=[], + development=False, + windows_sysprobe=False, + ) + + gen.assert_any_call(CORE_SCHEMA_FILE, "./cmd/agent/dist/datadog.yaml", "iot-agent", "linux") + + @patch("tasks.agent.refresh_assets") + @patch("tasks.agent.generate_template") + @patch("tasks.agent.sys") + def test_windows_skips_sysprobe_unless_requested(self, sys_mod, gen, _refresh): + sys_mod.platform = "win32" + ctx = MagicMock() + + generate_config_examples( + ctx, + flavor=AgentFlavor.base, + skip_assets=True, + build_tags=[], + development=False, + windows_sysprobe=False, + ) + + gen.assert_called_once_with(CORE_SCHEMA_FILE, "./cmd/agent/dist/datadog.yaml", "agent-py3", "windows") + + @patch("tasks.agent.refresh_assets") + @patch("tasks.agent.generate_template") + @patch("tasks.agent.sys") + def test_windows_with_sysprobe_renders_sysprobe(self, sys_mod, gen, _refresh): + sys_mod.platform = "win32" + ctx = MagicMock() + + generate_config_examples( + ctx, + flavor=AgentFlavor.base, + skip_assets=True, + build_tags=[], + development=False, + windows_sysprobe=True, + ) + + gen.assert_any_call(SYSPROBE_SCHEMA_FILE, "./cmd/agent/dist/system-probe.yaml", "system-probe", "windows") + self.assertEqual(gen.call_count, 2) + + @patch("tasks.agent.refresh_assets") + @patch("tasks.agent.generate_template") + @patch("tasks.agent.sys") + def test_skip_assets_false_calls_refresh_assets(self, sys_mod, _gen, refresh): + sys_mod.platform = "linux" + ctx = MagicMock() + + generate_config_examples( + ctx, + flavor=AgentFlavor.base, + skip_assets=False, + build_tags=["python"], + development=True, + windows_sysprobe=False, + ) + + refresh.assert_called_once_with( + ctx, ["python"], development=True, flavor=AgentFlavor.base.name, windows_sysprobe=False + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/unit_tests/cluster_agent_helpers_tests.py b/tasks/unit_tests/cluster_agent_helpers_tests.py new file mode 100644 index 000000000000..0d2e1cd9445e --- /dev/null +++ b/tasks/unit_tests/cluster_agent_helpers_tests.py @@ -0,0 +1,69 @@ +import unittest +from unittest.mock import MagicMock, patch + +from tasks.cluster_agent_helpers import build_common +from tasks.schema.template import CORE_SCHEMA_FILE + + +class TestClusterAgentHelpersBuildCommon(unittest.TestCase): + @patch("tasks.cluster_agent_helpers.schema_compress") + @patch("tasks.cluster_agent_helpers.refresh_assets_common") + @patch("tasks.cluster_agent_helpers.generate_template") + @patch("tasks.cluster_agent_helpers.go_build") + @patch("tasks.cluster_agent_helpers.get_build_flags") + def test_main_cluster_agent_renders_dca_template(self, gbf, _go, gen, _refresh, _compress): + gbf.return_value = ("ld", "gc", {}) + ctx = MagicMock() + + build_common( + ctx, + bin_path="./bin/datadog-cluster-agent", + build_tags=["kubeapiserver"], + bin_suffix="", + rebuild=False, + build_include=None, + build_exclude=None, + race=False, + development=False, + skip_assets=True, + ) + + gen.assert_called_once_with( + CORE_SCHEMA_FILE, + "./Dockerfiles/cluster-agent/datadog-cluster.yaml", + "dca", + "linux", + ) + + @patch("tasks.cluster_agent_helpers.schema_compress") + @patch("tasks.cluster_agent_helpers.refresh_assets_common") + @patch("tasks.cluster_agent_helpers.generate_template") + @patch("tasks.cluster_agent_helpers.go_build") + @patch("tasks.cluster_agent_helpers.get_build_flags") + def test_cloudfoundry_renders_dcacf_template(self, gbf, _go, gen, _refresh, _compress): + gbf.return_value = ("ld", "gc", {}) + ctx = MagicMock() + + build_common( + ctx, + bin_path="./bin/datadog-cluster-agent-cloudfoundry", + build_tags=["clusterchecks"], + bin_suffix="-cloudfoundry", + rebuild=False, + build_include=None, + build_exclude=None, + race=False, + development=False, + skip_assets=True, + ) + + gen.assert_called_once_with( + CORE_SCHEMA_FILE, + "./cloudfoundry.yaml", + "dcacf", + "linux", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/unit_tests/dogstatsd_tests.py b/tasks/unit_tests/dogstatsd_tests.py new file mode 100644 index 000000000000..63b3c4a75356 --- /dev/null +++ b/tasks/unit_tests/dogstatsd_tests.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch + +from invoke import MockContext + +from tasks.dogstatsd import DOGSTATSD_CONFIG_OUTPUT, build +from tasks.schema.template import CORE_SCHEMA_FILE + + +class TestDogstatsdBuild(unittest.TestCase): + @patch("tasks.dogstatsd.refresh_assets") + @patch("tasks.dogstatsd.generate_template") + @patch("tasks.dogstatsd.go_build") + @patch("tasks.dogstatsd.get_build_flags") + @patch("tasks.dogstatsd.sys") + def test_build_renders_dogstatsd_template(self, sys_mod, gbf, _go, gen, _refresh): + sys_mod.platform = "linux" + gbf.return_value = ("ld", "gc", {}) + ctx = MockContext() + + build(ctx) + + gen.assert_called_once_with( + CORE_SCHEMA_FILE, + DOGSTATSD_CONFIG_OUTPUT, + "dogstatsd", + "linux", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/unit_tests/schema_template_cli_tests.py b/tasks/unit_tests/schema_template_cli_tests.py new file mode 100644 index 000000000000..75626b8b635a --- /dev/null +++ b/tasks/unit_tests/schema_template_cli_tests.py @@ -0,0 +1,146 @@ +import os +import tempfile +import unittest + +import yaml + + +def _sample_core_schema(): + """Minimal enriched-schema shape sufficient to exercise generate_template. + + Mirrors the shape produced by `dda inv schema.generate` for the core + agent: a top-level `properties` dict whose entries each carry the + enrichment fields (visibility, description, tags, default, type). + Two settings is enough to verify both the per-entry rendering path + and the ordering driven by `properties` iteration. + """ + return { + "properties": { + "api_key": { + "node_type": "setting", + "type": "string", + "default": "", + "description": "Your Datadog API key", + "visibility": "public", + "tags": [], + }, + "site": { + "node_type": "setting", + "type": "string", + "default": "datadoghq.com", + "description": "The Datadog site URL", + "visibility": "public", + "tags": [], + }, + } + } + + +def _sample_sysprobe_schema(): + return { + "properties": { + "network_config": { + "node_type": "section", + "type": "object", + "description": "Network monitoring configuration", + "visibility": "public", + "tags": [], + "properties": { + "enabled": { + "node_type": "setting", + "type": "boolean", + "default": False, + "description": "Enable network monitoring", + "visibility": "public", + "tags": [], + }, + }, + }, + } + } + + +class TestSchemaTemplateCLI(unittest.TestCase): + def setUp(self): + self._tempdir = tempfile.TemporaryDirectory() + self.dir = self._tempdir.name + + def tearDown(self): + self._tempdir.cleanup() + + def _write_schema(self, schema, name="schema.yaml"): + path = os.path.join(self.dir, name) + with open(path, "w") as f: + yaml.safe_dump(schema, f, sort_keys=False) + return path + + def test_cli_main_writes_output(self): + from tasks.schema.template import main as cli_main + + schema_path = self._write_schema(_sample_core_schema()) + out_path = os.path.join(self.dir, "out.yaml") + + rc = cli_main(["template.py", schema_path, "agent-py3", "linux", out_path]) + + self.assertEqual(rc, 0) + self.assertTrue(os.path.isfile(out_path)) + with open(out_path) as f: + content = f.read() + self.assertIn("api_key", content) + self.assertIn("site", content) + self.assertIn("datadoghq.com", content) + + def test_cli_main_system_probe_build_type(self): + from tasks.schema.template import main as cli_main + + schema_path = self._write_schema(_sample_sysprobe_schema()) + out_path = os.path.join(self.dir, "out.yaml") + + rc = cli_main(["template.py", schema_path, "system-probe", "linux", out_path]) + + self.assertEqual(rc, 0) + with open(out_path) as f: + content = f.read() + self.assertIn("network_config", content) + self.assertIn("enabled", content) + + def test_cli_main_rejects_invalid_build_type(self): + from tasks.schema.template import main as cli_main + + schema_path = self._write_schema(_sample_core_schema()) + out_path = os.path.join(self.dir, "out.yaml") + + rc = cli_main(["template.py", schema_path, "bogus-build", "linux", out_path]) + + self.assertNotEqual(rc, 0) + + def test_cli_main_rejects_invalid_os_target(self): + from tasks.schema.template import main as cli_main + + schema_path = self._write_schema(_sample_core_schema()) + out_path = os.path.join(self.dir, "out.yaml") + + rc = cli_main(["template.py", schema_path, "agent-py3", "bogus-os", out_path]) + + self.assertNotEqual(rc, 0) + + def test_cli_main_rejects_missing_schema(self): + from tasks.schema.template import main as cli_main + + out_path = os.path.join(self.dir, "out.yaml") + missing = os.path.join(self.dir, "does-not-exist.yaml") + + rc = cli_main(["template.py", missing, "agent-py3", "linux", out_path]) + + self.assertNotEqual(rc, 0) + + def test_cli_main_rejects_wrong_argv_count(self): + from tasks.schema.template import main as cli_main + + rc = cli_main(["template.py", "only-one-arg"]) + + self.assertNotEqual(rc, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/winbuildscripts/Invoke-UnitTests.ps1 b/tasks/winbuildscripts/Invoke-UnitTests.ps1 index 7ab07310a652..30e28418a33f 100644 --- a/tasks/winbuildscripts/Invoke-UnitTests.ps1 +++ b/tasks/winbuildscripts/Invoke-UnitTests.ps1 @@ -88,7 +88,7 @@ Invoke-BuildScript ` New-LocalUser -Name "ddagentuser" -Description "Test user for the secrets feature on windows." -Password $Password } # Generate the datadog.yaml config file to be used in integration tests - & dda inv -- -e agent.generate-config --build-type="agent-py3" --output-file="./datadog.yaml" + & dda inv -- -e schema.template --schema=./pkg/config/schema/yaml/core_schema.yaml --build-type=agent-py3 --os-target=windows --output=./datadog.yaml # Build inputs needed for go builds & .\tasks\winbuildscripts\pre-go-build.ps1 }