| 1 | --- |
| 2 | # SPDX-License-Identifier: AGPL-3.0-or-later |
| 3 | # |
| 4 | # node_exporter (host metrics) + Grafana Alloy (scrape locally, |
| 5 | # remote_write to Grafana Cloud Mimir). Replaces the previous |
| 6 | # Promtail+node_exporter setup that scraped from a metal box. |
| 7 | # |
| 8 | # Operator side (one-time, see runbooks/observability.md): |
| 9 | # 1. Sign up for Grafana Cloud free tier at https://grafana.com/ |
| 10 | # 2. Create an Access Policy with `metrics:write` scope; mint a |
| 11 | # token under that policy. |
| 12 | # 3. From the Cloud portal, find the Prometheus instance details: |
| 13 | # - URL (e.g. https://prometheus-prod-NN-prod-us-central-0.grafana.net/api/prom/push) |
| 14 | # - Username (numeric tenant id, e.g. "1234567") |
| 15 | # 4. Set in inventory/production: |
| 16 | # grafana_cloud_prom_url: <URL above> |
| 17 | # grafana_cloud_prom_user: <numeric tenant id> |
| 18 | # grafana_cloud_prom_token: <token from step 2> |
| 19 | # 5. ansible-playbook ... -t monitoring (or just re-run site.yml) |
| 20 | |
| 21 | - name: node_exporter — install |
| 22 | apt: |
| 23 | name: prometheus-node-exporter |
| 24 | state: present |
| 25 | |
| 26 | - name: node_exporter — service started |
| 27 | systemd: |
| 28 | name: prometheus-node-exporter |
| 29 | state: started |
| 30 | enabled: yes |
| 31 | |
| 32 | # Grafana Alloy (the rebrand of Grafana Agent) ships from Grafana's |
| 33 | # apt repo. Add the repo + key idempotently. |
| 34 | - name: Grafana repo — keyring dir |
| 35 | file: |
| 36 | path: /etc/apt/keyrings |
| 37 | state: directory |
| 38 | mode: "0755" |
| 39 | |
| 40 | - name: Grafana repo — apt key |
| 41 | get_url: |
| 42 | url: https://apt.grafana.com/gpg.key |
| 43 | dest: /etc/apt/keyrings/grafana.gpg.key |
| 44 | mode: "0644" |
| 45 | |
| 46 | - name: Grafana repo — sources list |
| 47 | copy: |
| 48 | dest: /etc/apt/sources.list.d/grafana.list |
| 49 | content: | |
| 50 | deb [signed-by=/etc/apt/keyrings/grafana.gpg.key] https://apt.grafana.com stable main |
| 51 | mode: "0644" |
| 52 | register: grafana_repo |
| 53 | |
| 54 | - name: apt update if grafana repo just added |
| 55 | apt: |
| 56 | update_cache: yes |
| 57 | when: grafana_repo.changed |
| 58 | |
| 59 | - name: Grafana Alloy — install |
| 60 | apt: |
| 61 | name: alloy |
| 62 | state: present |
| 63 | |
| 64 | # Credentials live in a dedicated env file so the alloy systemd unit |
| 65 | # can pull them via EnvironmentFile=. Not in /etc/shithub because |
| 66 | # alloy is its own user; we don't want to widen group access there. |
| 67 | - name: Alloy creds dir |
| 68 | file: |
| 69 | path: /etc/alloy |
| 70 | state: directory |
| 71 | owner: root |
| 72 | group: alloy |
| 73 | mode: "0750" |
| 74 | |
| 75 | - name: Alloy credentials env file |
| 76 | template: |
| 77 | src: alloy.env.j2 |
| 78 | dest: /etc/alloy/credentials.env |
| 79 | owner: root |
| 80 | group: alloy |
| 81 | mode: "0640" |
| 82 | notify: restart alloy |
| 83 | |
| 84 | - name: Alloy config (River) |
| 85 | template: |
| 86 | src: alloy-config.river.j2 |
| 87 | dest: /etc/alloy/config.alloy |
| 88 | owner: root |
| 89 | group: alloy |
| 90 | mode: "0644" |
| 91 | notify: restart alloy |
| 92 | |
| 93 | - name: Alloy systemd drop-in dir |
| 94 | file: |
| 95 | path: /etc/systemd/system/alloy.service.d |
| 96 | state: directory |
| 97 | mode: "0755" |
| 98 | |
| 99 | # Override the upstream unit's EnvironmentFile so it picks up the |
| 100 | # credentials file above. Drop-in keeps the package-shipped unit |
| 101 | # intact across upgrades. |
| 102 | - name: Alloy systemd drop-in for credentials env |
| 103 | copy: |
| 104 | dest: /etc/systemd/system/alloy.service.d/shithub.conf |
| 105 | content: | |
| 106 | [Service] |
| 107 | EnvironmentFile=/etc/alloy/credentials.env |
| 108 | mode: "0644" |
| 109 | notify: [daemon-reload, restart alloy] |
| 110 | |
| 111 | - name: Alloy enabled + started |
| 112 | systemd: |
| 113 | name: alloy |
| 114 | state: started |
| 115 | enabled: yes |
| 116 |