forked from linux/documentation
update sample config
This commit is contained in:
bruhn_b committed
1 parent
b6c72b3845
commit
f5e8357db9
1 file changed
+366
-178
@@ -10,250 +10,438 @@ The config consists of following components:
|
||||
### Config
|
||||
|
||||
```
|
||||
// ##########################
|
||||
// #### Write Components ####
|
||||
// ##########################
|
||||
prometheus.remote_write "default" {
|
||||
endpoint {
|
||||
url = "https://mimir-test.psi.ch/api/v1/push"
|
||||
basic_auth {
|
||||
username = "lx-collector"
|
||||
password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
|
||||
}
|
||||
}
|
||||
}
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "https://loki-test.psi.ch/loki/api/v1/push"
|
||||
basic_auth {
|
||||
username = "lx-collector"
|
||||
password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
|
||||
}
|
||||
}
|
||||
}
|
||||
otelcol.exporter.otlp "default" {
|
||||
client {
|
||||
endpoint = "tempo-test.psi.ch:4317"
|
||||
auth = otelcol.auth.basic.default.handler
|
||||
}
|
||||
}
|
||||
otelcol.auth.basic "default" {
|
||||
username = "lx-collector"
|
||||
password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
|
||||
```alloy
|
||||
// ============================================================================
|
||||
// Grafana Alloy production starter config
|
||||
// Target version: grafana/alloy:v1.16.3
|
||||
//
|
||||
// Collects:
|
||||
// - Alloy self-monitoring metrics
|
||||
// - Linux host metrics
|
||||
// - Docker container metrics
|
||||
// - Docker container logs
|
||||
// - systemd journal logs
|
||||
// - Optional Prometheus metric endpoints
|
||||
// - Optional OTLP traces
|
||||
//
|
||||
// Sends:
|
||||
// - Metrics -> Mimir
|
||||
// - Logs -> Loki
|
||||
// - Traces -> Tempo
|
||||
//
|
||||
// Multitenancy is assigned by NGINX:
|
||||
// Basic Auth username -> X-Scope-OrgID
|
||||
// ============================================================================
|
||||
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
|
||||
// #####################################
|
||||
// #### Docker Metrics Configuration ####
|
||||
// ######################################
|
||||
livedebugging {}
|
||||
|
||||
// Host Cadvisor on the Docker socket to expose container metrics.
|
||||
prometheus.exporter.cadvisor "docker" {
|
||||
docker_only = true
|
||||
}
|
||||
|
||||
discovery.relabel "example" {
|
||||
targets = prometheus.exporter.cadvisor.docker.targets
|
||||
// ============================================================================
|
||||
// Remote destinations
|
||||
// ============================================================================
|
||||
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "integrations/docker"
|
||||
prometheus.remote_write "mimir" {
|
||||
endpoint {
|
||||
url = "https://mimir.psi.ch/api/v1/push"
|
||||
|
||||
basic_auth {
|
||||
username = "lx-collector"
|
||||
password = "SECRET"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
queue_config {
|
||||
capacity = 10000
|
||||
min_shards = 1
|
||||
max_shards = 8
|
||||
max_samples_per_send = 2000
|
||||
batch_send_deadline = "5s"
|
||||
retry_on_http_429 = true
|
||||
}
|
||||
}
|
||||
|
||||
// Configure a prometheus.scrape component to collect cadvisor metrics.
|
||||
prometheus.scrape "scraper" {
|
||||
targets = discovery.relabel.example.output
|
||||
forward_to = [ prometheus.remote_write.default.receiver ]
|
||||
|
||||
|
||||
scrape_interval = "10s"
|
||||
}
|
||||
|
||||
// ######################################
|
||||
// #### Docker Logging Configuration ####
|
||||
// ######################################
|
||||
|
||||
// Discover Docker containers and extract metadata.
|
||||
discovery.docker "linux" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
|
||||
// Define a relabeling rule to create a service name from the container name.
|
||||
discovery.relabel "logs_integrations_docker" {
|
||||
targets = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container_name"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Configure a loki.source.docker component to collect logs from Docker containers.
|
||||
loki.source.docker "default" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.docker.linux.targets
|
||||
relabel_rules = discovery.relabel.logs_integrations_docker.rules
|
||||
forward_to = [loki.write.default.receiver]
|
||||
wal {
|
||||
truncate_frequency = "2h"
|
||||
min_keepalive_time = "5m"
|
||||
max_keepalive_time = "12h"
|
||||
}
|
||||
}
|
||||
|
||||
// ###############################
|
||||
// #### Metrics Configuration ####
|
||||
// ###############################
|
||||
// This block relabels metrics coming from node_exporter to add standard labels
|
||||
discovery.relabel "integrations_node_exporter" {
|
||||
targets = prometheus.exporter.unix.integrations_node_exporter.targets
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "https://loki.psi.ch/loki/api/v1/push"
|
||||
|
||||
basic_auth {
|
||||
username = "lx-collector"
|
||||
password = "SECRET"
|
||||
}
|
||||
}
|
||||
|
||||
external_labels = {
|
||||
collector = "alloy",
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.auth.basic "tempo" {
|
||||
username = "lx-collector"
|
||||
password = "SECRET"
|
||||
}
|
||||
|
||||
otelcol.exporter.otlp "tempo" {
|
||||
client {
|
||||
endpoint = "tempo.psi.ch:4317"
|
||||
auth = otelcol.auth.basic.tempo.handler
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// Alloy self-monitoring metrics
|
||||
// ============================================================================
|
||||
|
||||
prometheus.exporter.self "alloy" {}
|
||||
|
||||
discovery.relabel "alloy_self_metrics" {
|
||||
targets = prometheus.exporter.self.alloy.targets
|
||||
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "integrations/self"
|
||||
}
|
||||
|
||||
rule {
|
||||
// Set the instance label to the hostname of the machine
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
|
||||
rule {
|
||||
// Set a standard job name for all node_exporter metrics
|
||||
target_label = "job"
|
||||
replacement = "integrations/node_exporter"
|
||||
}
|
||||
}
|
||||
|
||||
// Configure the node_exporter integration to collect system metrics
|
||||
prometheus.exporter.unix "integrations_node_exporter" {
|
||||
// Disable unnecessary collectors to reduce overhead
|
||||
disable_collectors = ["ipvs", "btrfs", "infiniband", "zfs"]
|
||||
enable_collectors = ["meminfo", "xfs"]
|
||||
prometheus.scrape "alloy_self" {
|
||||
targets = discovery.relabel.alloy_self_metrics.output
|
||||
scrape_interval = "30s"
|
||||
scrape_timeout = "10s"
|
||||
|
||||
forward_to = [
|
||||
prometheus.remote_write.mimir.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// Linux host metrics
|
||||
// ============================================================================
|
||||
|
||||
prometheus.exporter.unix "node" {
|
||||
include_exporter_metrics = false
|
||||
|
||||
disable_collectors = [
|
||||
"ipvs",
|
||||
"btrfs",
|
||||
"infiniband",
|
||||
"zfs",
|
||||
]
|
||||
|
||||
enable_collectors = [
|
||||
"meminfo",
|
||||
"xfs",
|
||||
]
|
||||
|
||||
filesystem {
|
||||
// Exclude filesystem types that aren't relevant for monitoring
|
||||
fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
|
||||
// Exclude mount points that aren't relevant for monitoring
|
||||
fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
|
||||
|
||||
mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
|
||||
// Timeout for filesystem operations
|
||||
mount_timeout = "5s"
|
||||
|
||||
mount_timeout = "5s"
|
||||
}
|
||||
|
||||
netclass {
|
||||
// Ignore virtual and container network interfaces
|
||||
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
|
||||
netdev {
|
||||
// Exclude virtual and container network interfaces from device metrics
|
||||
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
// Define how to scrape metrics from the node_exporter
|
||||
prometheus.scrape "integrations_node_exporter" {
|
||||
scrape_interval = "15s"
|
||||
// Use the targets with labels from the discovery.relabel component
|
||||
targets = discovery.relabel.integrations_node_exporter.output
|
||||
// Send the scraped metrics to the relabeling component
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
discovery.relabel "node_metrics" {
|
||||
targets = prometheus.exporter.unix.node.targets
|
||||
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "integrations/node_exporter"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
}
|
||||
|
||||
// ############################
|
||||
// #### Logs Configuration ####
|
||||
// ############################
|
||||
prometheus.scrape "node" {
|
||||
targets = discovery.relabel.node_metrics.output
|
||||
scrape_interval = "60s"
|
||||
scrape_timeout = "10s"
|
||||
|
||||
// Collect logs from systemd journal for node_exporter integration
|
||||
loki.source.journal "logs_integrations_integrations_node_exporter_journal_scrape" {
|
||||
// Only collect logs from the last 24 hours
|
||||
max_age = "24h0m0s"
|
||||
// Apply relabeling rules to the logs
|
||||
relabel_rules = discovery.relabel.logs_integrations_integrations_node_exporter_journal_scrape.rules
|
||||
// Send logs to the local Loki instance
|
||||
forward_to = [loki.write.default.receiver]
|
||||
forward_to = [
|
||||
prometheus.remote_write.mimir.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
// Define which log files to collect for node_exporter
|
||||
local.file_match "logs_integrations_integrations_node_exporter_direct_scrape" {
|
||||
path_targets = [{
|
||||
// Target localhost for log collection
|
||||
__address__ = "localhost",
|
||||
// Collect standard system logs
|
||||
__path__ = "/var/log/{syslog,messages,*.log}",
|
||||
// Add instance label with hostname
|
||||
instance = constants.hostname,
|
||||
// Add job label for logs
|
||||
job = "integrations/node_exporter",
|
||||
}]
|
||||
|
||||
// ============================================================================
|
||||
// Docker container metrics
|
||||
// ============================================================================
|
||||
|
||||
prometheus.exporter.cadvisor "docker" {
|
||||
docker_only = true
|
||||
store_container_labels = false
|
||||
}
|
||||
|
||||
// Define relabeling rules for systemd journal logs
|
||||
discovery.relabel "logs_integrations_integrations_node_exporter_journal_scrape" {
|
||||
discovery.relabel "docker_metrics" {
|
||||
targets = prometheus.exporter.cadvisor.docker.targets
|
||||
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "integrations/docker"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "docker" {
|
||||
targets = discovery.relabel.docker_metrics.output
|
||||
scrape_interval = "60s"
|
||||
scrape_timeout = "10s"
|
||||
|
||||
forward_to = [
|
||||
prometheus.remote_write.mimir.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// Docker container logs
|
||||
// ============================================================================
|
||||
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
refresh_interval = "60s"
|
||||
}
|
||||
|
||||
discovery.relabel "docker_logs" {
|
||||
// This component exports relabeling rules only.
|
||||
targets = []
|
||||
|
||||
rule {
|
||||
// Extract systemd unit information into a label
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container_name"
|
||||
replacement = "$1"
|
||||
}
|
||||
|
||||
// Stable Docker Compose service name, when available.
|
||||
rule {
|
||||
source_labels = [
|
||||
"__meta_docker_container_label_com_docker_compose_service",
|
||||
]
|
||||
|
||||
target_label = "service_name"
|
||||
}
|
||||
|
||||
rule {
|
||||
// Extract boot ID information into a label
|
||||
source_labels = ["__journal__boot_id"]
|
||||
target_label = "boot_id"
|
||||
target_label = "job"
|
||||
replacement = "integrations/docker"
|
||||
}
|
||||
|
||||
rule {
|
||||
// Extract transport information into a label
|
||||
target_label = "hostname"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.docker.containers.targets
|
||||
relabel_rules = discovery.relabel.docker_logs.rules
|
||||
|
||||
forward_to = [
|
||||
loki.write.default.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// systemd journal logs
|
||||
//
|
||||
// Cardinality protection:
|
||||
//
|
||||
// - boot_id is NOT promoted to a Loki label.
|
||||
// - PID, UID, GID and session identifiers are NOT promoted.
|
||||
// - session-*.scope is NOT promoted.
|
||||
// - user-*.slice and transient scopes are NOT promoted.
|
||||
// - Only explicitly allowlisted, stable service units become labels.
|
||||
//
|
||||
// Logs from non-allowlisted units are still collected. They simply do not get
|
||||
// a systemd_unit label.
|
||||
// ============================================================================
|
||||
|
||||
loki.relabel "journal_logs" {
|
||||
// This component exports relabeling rules only.
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "integrations/systemd_journal"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "hostname"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
|
||||
// Small, bounded label value set.
|
||||
rule {
|
||||
source_labels = ["__journal__transport"]
|
||||
target_label = "transport"
|
||||
}
|
||||
|
||||
// Small, bounded label value set.
|
||||
rule {
|
||||
// Extract log priority into a level label
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
|
||||
// Only stable service units are promoted to labels.
|
||||
//
|
||||
// Add additional long-lived services here as required.
|
||||
// Do NOT add patterns such as:
|
||||
//
|
||||
// session-.*
|
||||
// user-.*
|
||||
// .*\\.scope
|
||||
//
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
|
||||
regex = "(alloy\\.service|sshd\\.service|docker\\.service|containerd\\.service|nginx\\.service|httpd\\.service|gitea\\.service|postgresql.*\\.service|slurmd\\.service|slurmctld\\.service|slurmdbd\\.service|chronyd\\.service)"
|
||||
|
||||
target_label = "systemd_unit"
|
||||
}
|
||||
}
|
||||
|
||||
// Collect logs from files for node_exporter
|
||||
loki.source.file "logs_integrations_integrations_node_exporter_direct_scrape" {
|
||||
// Use targets defined in local.file_match
|
||||
targets = local.file_match.logs_integrations_integrations_node_exporter_direct_scrape.targets
|
||||
// Send logs to the local Loki instance
|
||||
forward_to = [loki.write.default.receiver]
|
||||
loki.source.journal "systemd" {
|
||||
max_age = "12h"
|
||||
relabel_rules = loki.relabel.journal_logs.rules
|
||||
|
||||
labels = {
|
||||
component = "systemd_journal",
|
||||
service_name = "systemd_journal",
|
||||
}
|
||||
|
||||
forward_to = [
|
||||
loki.write.default.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
// #######################
|
||||
// #### Gitea Metrics ####
|
||||
// #######################
|
||||
|
||||
// Enable live debugging features (empty config means use defaults)
|
||||
livedebugging{}
|
||||
// ============================================================================
|
||||
// Optional custom Prometheus endpoints
|
||||
//
|
||||
// Add application and webserver exporters here.
|
||||
//
|
||||
// Examples:
|
||||
//
|
||||
// NGINX exporter:
|
||||
// localhost:9113
|
||||
//
|
||||
// Apache exporter:
|
||||
// localhost:9117
|
||||
//
|
||||
// Application exporter:
|
||||
// localhost:8080
|
||||
// ============================================================================
|
||||
|
||||
discovery.relabel "metrics_integrations_integrations_gitea" {
|
||||
targets = [{
|
||||
__address__ = "gitea.psi.ch",
|
||||
__scheme__ = "https",
|
||||
__metrics_path__ = "/metrics",
|
||||
}]
|
||||
discovery.relabel "custom_metrics" {
|
||||
targets = [
|
||||
// Uncomment and adjust as needed:
|
||||
//
|
||||
// {
|
||||
// __address__ = "localhost:9113",
|
||||
// job = "integrations/nginx",
|
||||
// },
|
||||
//
|
||||
// {
|
||||
// __address__ = "localhost:9117",
|
||||
// job = "integrations/apache",
|
||||
// },
|
||||
//
|
||||
// {
|
||||
// __address__ = "localhost:8080",
|
||||
// job = "integrations/application",
|
||||
// },
|
||||
]
|
||||
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = constants.hostname
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "metrics_integrations_integrations_gitea" {
|
||||
targets = discovery.relabel.metrics_integrations_integrations_gitea.output
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
job_name = "integrations/gitea"
|
||||
prometheus.scrape "custom_metrics" {
|
||||
targets = discovery.relabel.custom_metrics.output
|
||||
scrape_interval = "30s"
|
||||
scrape_timeout = "10s"
|
||||
|
||||
forward_to = [
|
||||
prometheus.remote_write.mimir.receiver,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// Optional OTLP trace receiver
|
||||
//
|
||||
// Applications can send traces to this Alloy instance using:
|
||||
//
|
||||
// OTLP/gRPC: localhost:4317
|
||||
// OTLP/HTTP: localhost:4318
|
||||
//
|
||||
// When Alloy runs in Docker, publish these ports or use host networking.
|
||||
// ============================================================================
|
||||
|
||||
otelcol.receiver.otlp "default" {
|
||||
grpc {
|
||||
endpoint = "0.0.0.0:4317"
|
||||
}
|
||||
|
||||
http {
|
||||
endpoint = "0.0.0.0:4318"
|
||||
}
|
||||
|
||||
output {
|
||||
traces = [
|
||||
otelcol.processor.batch.traces.input,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.processor.batch "traces" {
|
||||
timeout = "5s"
|
||||
send_batch_size = 8192
|
||||
|
||||
output {
|
||||
traces = [
|
||||
otelcol.exporter.otlp.tempo.input,
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
Reference in new issue
Block a user