From f5e8357db98c6d90ba1519d923bd11fffa26ddd2 Mon Sep 17 00:00:00 2001
From: bruhn_b <basil.bruhn@psi.ch>
Date: Fri, 17 Jul 2026 10:03:37 +0200
Subject: [PATCH] update sample config

---
 docs/engineering-guide/alloy_sample_config.md | 544 ++++++++++++------
 1 file changed, 366 insertions(+), 178 deletions(-)

diff --git a/docs/engineering-guide/alloy_sample_config.md b/docs/engineering-guide/alloy_sample_config.md
index 7c61a5c0..dc37d346 100644
--- a/docs/engineering-guide/alloy_sample_config.md
+++ b/docs/engineering-guide/alloy_sample_config.md
@@ -10,250 +10,438 @@ The config consists of following components:
 ### Config
 
 ```
-// ##########################
-// #### Write Components ####
-// ##########################
-prometheus.remote_write "default" {
-	endpoint {
-		url = "https://mimir-test.psi.ch/api/v1/push"
-		basic_auth {
-			username = "lx-collector"
-			password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
-		}
-	}
-}
-loki.write "default" {
-	endpoint {
-		url = "https://loki-test.psi.ch/loki/api/v1/push"
-		basic_auth {
-			username = "lx-collector"
-			password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
-		}
-	}
-}
-otelcol.exporter.otlp "default" {
-	client {
-		endpoint = "tempo-test.psi.ch:4317"
-		auth     = otelcol.auth.basic.default.handler
-	}
-}
-otelcol.auth.basic "default" {
-	username = "lx-collector"
-	password = "SUPER_SECRET_PLAINTEXT_PASSWORD"
+```alloy
+// ============================================================================
+// Grafana Alloy production starter config
+// Target version: grafana/alloy:v1.16.3
+//
+// Collects:
+//   - Alloy self-monitoring metrics
+//   - Linux host metrics
+//   - Docker container metrics
+//   - Docker container logs
+//   - systemd journal logs
+//   - Optional Prometheus metric endpoints
+//   - Optional OTLP traces
+//
+// Sends:
+//   - Metrics -> Mimir
+//   - Logs    -> Loki
+//   - Traces  -> Tempo
+//
+// Multitenancy is assigned by NGINX:
+//   Basic Auth username -> X-Scope-OrgID
+// ============================================================================
+
+logging {
+  level  = "info"
+  format = "logfmt"
 }
 
-// #####################################
-// #### Docker Metrics Configuration ####
-// ######################################
+livedebugging {}
 
-// Host Cadvisor on the Docker socket to expose container metrics.
-prometheus.exporter.cadvisor "docker" {
-  docker_only = true
-}
 
-discovery.relabel "example" {
-    targets = prometheus.exporter.cadvisor.docker.targets
+// ============================================================================
+// Remote destinations
+// ============================================================================
 
-    rule {
-        target_label = "job"
-        replacement  = "integrations/docker"
+prometheus.remote_write "mimir" {
+  endpoint {
+    url = "https://mimir.psi.ch/api/v1/push"
+
+    basic_auth {
+      username = "lx-collector"
+      password = "SECRET"
     }
 
-    rule {
-        target_label = "instance"
-        replacement  = constants.hostname
+    queue_config {
+      capacity             = 10000
+      min_shards           = 1
+      max_shards           = 8
+      max_samples_per_send = 2000
+      batch_send_deadline  = "5s"
+      retry_on_http_429    = true
     }
-}
-
-// Configure a prometheus.scrape component to collect cadvisor metrics.
-prometheus.scrape "scraper" {
-  targets    = discovery.relabel.example.output
-  forward_to = [ prometheus.remote_write.default.receiver ]
-
-
-  scrape_interval = "10s"
-}
-
-// ######################################
-// #### Docker Logging Configuration ####
-// ######################################
-
-// Discover Docker containers and extract metadata.
-discovery.docker "linux" {
-  host = "unix:///var/run/docker.sock"
-}
-
-// Define a relabeling rule to create a service name from the container name.
-discovery.relabel "logs_integrations_docker" {
-      targets = []
-  
-      rule {
-          source_labels = ["__meta_docker_container_name"]
-          regex = "/(.*)"
-          target_label = "container_name"
-      }
-
-     rule {
-        target_label = "instance"
-        replacement  = constants.hostname
-    }
-
   }
 
-// Configure a loki.source.docker component to collect logs from Docker containers.
-loki.source.docker "default" {
-  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.docker.linux.targets
-  relabel_rules = discovery.relabel.logs_integrations_docker.rules
-  forward_to = [loki.write.default.receiver]
+  wal {
+    truncate_frequency  = "2h"
+    min_keepalive_time  = "5m"
+    max_keepalive_time  = "12h"
+  }
 }
 
-// ###############################
-// #### Metrics Configuration ####
-// ###############################
-// This block relabels metrics coming from node_exporter to add standard labels
-discovery.relabel "integrations_node_exporter" {
-  targets = prometheus.exporter.unix.integrations_node_exporter.targets
+loki.write "default" {
+  endpoint {
+    url = "https://loki.psi.ch/loki/api/v1/push"
+
+    basic_auth {
+      username = "lx-collector"
+      password = "SECRET"
+    }
+  }
+
+  external_labels = {
+    collector = "alloy",
+  }
+}
+
+otelcol.auth.basic "tempo" {
+  username = "lx-collector"
+  password = "SECRET"
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo.psi.ch:4317"
+    auth     = otelcol.auth.basic.tempo.handler
+  }
+}
+
+
+// ============================================================================
+// Alloy self-monitoring metrics
+// ============================================================================
+
+prometheus.exporter.self "alloy" {}
+
+discovery.relabel "alloy_self_metrics" {
+  targets = prometheus.exporter.self.alloy.targets
+
+  rule {
+    target_label = "job"
+    replacement  = "integrations/self"
+  }
 
   rule {
-    // Set the instance label to the hostname of the machine
     target_label = "instance"
     replacement  = constants.hostname
   }
-
-  rule {
-    // Set a standard job name for all node_exporter metrics
-    target_label = "job"
-    replacement = "integrations/node_exporter"
-  }
 }
 
-// Configure the node_exporter integration to collect system metrics
-prometheus.exporter.unix "integrations_node_exporter" {
-  // Disable unnecessary collectors to reduce overhead
-  disable_collectors = ["ipvs", "btrfs", "infiniband", "zfs"]
-  enable_collectors = ["meminfo", "xfs"]
+prometheus.scrape "alloy_self" {
+  targets         = discovery.relabel.alloy_self_metrics.output
+  scrape_interval = "30s"
+  scrape_timeout  = "10s"
+
+  forward_to = [
+    prometheus.remote_write.mimir.receiver,
+  ]
+}
+
+
+// ============================================================================
+// Linux host metrics
+// ============================================================================
+
+prometheus.exporter.unix "node" {
+  include_exporter_metrics = false
+
+  disable_collectors = [
+    "ipvs",
+    "btrfs",
+    "infiniband",
+    "zfs",
+  ]
+
+  enable_collectors = [
+    "meminfo",
+    "xfs",
+  ]
 
   filesystem {
-    // Exclude filesystem types that aren't relevant for monitoring
-    fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
-    // Exclude mount points that aren't relevant for monitoring
+    fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
+
     mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
-    // Timeout for filesystem operations
-    mount_timeout        = "5s"
+
+    mount_timeout = "5s"
   }
 
   netclass {
-    // Ignore virtual and container network interfaces
     ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
   }
 
   netdev {
-    // Exclude virtual and container network interfaces from device metrics
     device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
   }
-
-
 }
 
-// Define how to scrape metrics from the node_exporter
-prometheus.scrape "integrations_node_exporter" {
-scrape_interval = "15s"
-  // Use the targets with labels from the discovery.relabel component
-  targets    = discovery.relabel.integrations_node_exporter.output
-  // Send the scraped metrics to the relabeling component
-  forward_to = [prometheus.remote_write.default.receiver]
+discovery.relabel "node_metrics" {
+  targets = prometheus.exporter.unix.node.targets
+
+  rule {
+    target_label = "job"
+    replacement  = "integrations/node_exporter"
+  }
+
+  rule {
+    target_label = "instance"
+    replacement  = constants.hostname
+  }
 }
 
-// ############################
-// #### Logs Configuration ####
-// ############################
+prometheus.scrape "node" {
+  targets         = discovery.relabel.node_metrics.output
+  scrape_interval = "60s"
+  scrape_timeout  = "10s"
 
-// Collect logs from systemd journal for node_exporter integration
-loki.source.journal "logs_integrations_integrations_node_exporter_journal_scrape" {
-  // Only collect logs from the last 24 hours
-  max_age       = "24h0m0s"
-  // Apply relabeling rules to the logs
-  relabel_rules = discovery.relabel.logs_integrations_integrations_node_exporter_journal_scrape.rules
-  // Send logs to the local Loki instance
-  forward_to    = [loki.write.default.receiver]
+  forward_to = [
+    prometheus.remote_write.mimir.receiver,
+  ]
 }
 
-// Define which log files to collect for node_exporter
-local.file_match "logs_integrations_integrations_node_exporter_direct_scrape" {
-  path_targets = [{
-    // Target localhost for log collection
-    __address__ = "localhost",
-    // Collect standard system logs
-    __path__    = "/var/log/{syslog,messages,*.log}",
-    // Add instance label with hostname
-    instance    = constants.hostname,
-    // Add job label for logs
-    job         = "integrations/node_exporter",
-  }]
+
+// ============================================================================
+// Docker container metrics
+// ============================================================================
+
+prometheus.exporter.cadvisor "docker" {
+  docker_only            = true
+  store_container_labels = false
 }
 
-// Define relabeling rules for systemd journal logs
-discovery.relabel "logs_integrations_integrations_node_exporter_journal_scrape" {
+discovery.relabel "docker_metrics" {
+  targets = prometheus.exporter.cadvisor.docker.targets
+
+  rule {
+    target_label = "job"
+    replacement  = "integrations/docker"
+  }
+
+  rule {
+    target_label = "instance"
+    replacement  = constants.hostname
+  }
+}
+
+prometheus.scrape "docker" {
+  targets         = discovery.relabel.docker_metrics.output
+  scrape_interval = "60s"
+  scrape_timeout  = "10s"
+
+  forward_to = [
+    prometheus.remote_write.mimir.receiver,
+  ]
+}
+
+
+// ============================================================================
+// Docker container logs
+// ============================================================================
+
+discovery.docker "containers" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "60s"
+}
+
+discovery.relabel "docker_logs" {
+  // This component exports relabeling rules only.
   targets = []
 
   rule {
-    // Extract systemd unit information into a label
-    source_labels = ["__journal__systemd_unit"]
-    target_label  = "unit"
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+    replacement   = "$1"
+  }
+
+  // Stable Docker Compose service name, when available.
+  rule {
+    source_labels = [
+      "__meta_docker_container_label_com_docker_compose_service",
+    ]
+
+    target_label = "service_name"
   }
 
   rule {
-    // Extract boot ID information into a label
-    source_labels = ["__journal__boot_id"]
-    target_label  = "boot_id"
+    target_label = "job"
+    replacement  = "integrations/docker"
   }
 
   rule {
-    // Extract transport information into a label
+    target_label = "hostname"
+    replacement  = constants.hostname
+  }
+}
+
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.docker.containers.targets
+  relabel_rules = discovery.relabel.docker_logs.rules
+
+  forward_to = [
+    loki.write.default.receiver,
+  ]
+}
+
+
+// ============================================================================
+// systemd journal logs
+//
+// Cardinality protection:
+//
+//   - boot_id is NOT promoted to a Loki label.
+//   - PID, UID, GID and session identifiers are NOT promoted.
+//   - session-*.scope is NOT promoted.
+//   - user-*.slice and transient scopes are NOT promoted.
+//   - Only explicitly allowlisted, stable service units become labels.
+//
+// Logs from non-allowlisted units are still collected. They simply do not get
+// a systemd_unit label.
+// ============================================================================
+
+loki.relabel "journal_logs" {
+  // This component exports relabeling rules only.
+  forward_to = []
+
+  rule {
+    target_label = "job"
+    replacement  = "integrations/systemd_journal"
+  }
+
+  rule {
+    target_label = "hostname"
+    replacement  = constants.hostname
+  }
+
+  // Small, bounded label value set.
+  rule {
     source_labels = ["__journal__transport"]
     target_label  = "transport"
   }
 
+  // Small, bounded label value set.
   rule {
-    // Extract log priority into a level label
     source_labels = ["__journal_priority_keyword"]
     target_label  = "level"
   }
+
+  // Only stable service units are promoted to labels.
+  //
+  // Add additional long-lived services here as required.
+  // Do NOT add patterns such as:
+  //
+  //   session-.*
+  //   user-.*
+  //   .*\\.scope
+  //
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+
+    regex = "(alloy\\.service|sshd\\.service|docker\\.service|containerd\\.service|nginx\\.service|httpd\\.service|gitea\\.service|postgresql.*\\.service|slurmd\\.service|slurmctld\\.service|slurmdbd\\.service|chronyd\\.service)"
+
+    target_label = "systemd_unit"
+  }
 }
 
-// Collect logs from files for node_exporter
-loki.source.file "logs_integrations_integrations_node_exporter_direct_scrape" {
-  // Use targets defined in local.file_match
-  targets    = local.file_match.logs_integrations_integrations_node_exporter_direct_scrape.targets
-  // Send logs to the local Loki instance
-  forward_to = [loki.write.default.receiver]
+loki.source.journal "systemd" {
+  max_age       = "12h"
+  relabel_rules = loki.relabel.journal_logs.rules
+
+  labels = {
+    component    = "systemd_journal",
+    service_name = "systemd_journal",
+  }
+
+  forward_to = [
+    loki.write.default.receiver,
+  ]
 }
 
-// #######################
-// #### Gitea Metrics ####
-// #######################
 
-// Enable live debugging features (empty config means use defaults)
-livedebugging{}
+// ============================================================================
+// Optional custom Prometheus endpoints
+//
+// Add application and webserver exporters here.
+//
+// Examples:
+//
+//   NGINX exporter:
+//     localhost:9113
+//
+//   Apache exporter:
+//     localhost:9117
+//
+//   Application exporter:
+//     localhost:8080
+// ============================================================================
 
-discovery.relabel "metrics_integrations_integrations_gitea" {
-	targets = [{
-		__address__ = "gitea.psi.ch",
-		__scheme__ = "https",
-		__metrics_path__ = "/metrics",
-	}]
+discovery.relabel "custom_metrics" {
+  targets = [
+    // Uncomment and adjust as needed:
+    //
+    // {
+    //   __address__ = "localhost:9113",
+    //   job         = "integrations/nginx",
+    // },
+    //
+    // {
+    //   __address__ = "localhost:9117",
+    //   job         = "integrations/apache",
+    // },
+    //
+    // {
+    //   __address__ = "localhost:8080",
+    //   job         = "integrations/application",
+    // },
+  ]
 
-	rule {
-		target_label = "instance"
-		replacement  = constants.hostname
-	}
+  rule {
+    target_label = "instance"
+    replacement  = constants.hostname
+  }
 }
 
-prometheus.scrape "metrics_integrations_integrations_gitea" {
-	targets    = discovery.relabel.metrics_integrations_integrations_gitea.output
-	forward_to = [prometheus.remote_write.default.receiver]
-	job_name   = "integrations/gitea"
+prometheus.scrape "custom_metrics" {
+  targets         = discovery.relabel.custom_metrics.output
+  scrape_interval = "30s"
+  scrape_timeout  = "10s"
+
+  forward_to = [
+    prometheus.remote_write.mimir.receiver,
+  ]
+}
+
+
+// ============================================================================
+// Optional OTLP trace receiver
+//
+// Applications can send traces to this Alloy instance using:
+//
+//   OTLP/gRPC: localhost:4317
+//   OTLP/HTTP: localhost:4318
+//
+// When Alloy runs in Docker, publish these ports or use host networking.
+// ============================================================================
+
+otelcol.receiver.otlp "default" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+
+  http {
+    endpoint = "0.0.0.0:4318"
+  }
+
+  output {
+    traces = [
+      otelcol.processor.batch.traces.input,
+    ]
+  }
+}
+
+otelcol.processor.batch "traces" {
+  timeout         = "5s"
+  send_batch_size = 8192
+
+  output {
+    traces = [
+      otelcol.exporter.otlp.tempo.input,
+    ]
+  }
 }
 ```
 
+```
+