#!/bin/sh # Starter alerts for the Grafana stack in https://exe.dev/deploy/grafana: # an "Email me" contact point that mails the VM's owner through exe.dev's # email gateway (no SMTP), a notification policy that uses it, and five # alert rules in the "VM alerts" folder. Everything stays editable in # Grafana's UI. Run it in ~/monitoring once Grafana is up; it is safe to # run again (it overwrites these five rules and the contact point). set -eu cd "$(dirname "$0")" OWNER="$(sed -n 's/^GF_SECURITY_ADMIN_USER=//p' grafana.env)" G=http://localhost:8000 # Grafana trusts X-ExeDev-Email from anything that reaches port 8000; on # this VM that is exe.dev's proxy and local processes, like this script. api() { method=$1 path=$2; shift 2 curl -fsS -X "$method" "$G$path" -H "X-ExeDev-Email: $OWNER" \ -H "X-Disable-Provenance: true" -H "Content-Type: application/json" "$@" } for i in $(seq 60); do curl -fsS "$G/api/health" >/dev/null 2>&1 && break sleep 2 done # 1. Contact point: exe.dev's email gateway, to the owner. cp=$(jq -n --arg to "$OWNER" '{ uid: "email-me", name: "Email me", type: "webhook", settings: { url: "http://169.254.169.254/gateway/email/send", httpMethod: "POST", payload: { template: "{{ coll.Dict \"to\" .Vars.to \"subject\" (tmpl.Exec \"default.title\" .) \"body\" (tmpl.Exec \"default.message\" .) | data.ToJSON }}", vars: {to: $to} } } }') api PUT /api/v1/provisioning/contact-points/email-me -d "$cp" >/dev/null 2>&1 || api POST /api/v1/provisioning/contact-points -d "$cp" >/dev/null # 2. Send every alert to it, grouped by rule. api PUT /api/v1/provisioning/policies -d '{ "receiver": "Email me", "group_by": ["grafana_folder", "alertname"], "group_wait": "30s", "group_interval": "5m", "repeat_interval": "12h" }' >/dev/null # 3. A folder and five rules. api GET /api/folders/vm-alerts >/dev/null 2>&1 || api POST /api/folders -d '{"uid": "vm-alerts", "title": "VM alerts"}' >/dev/null rule() { # uid title for expr op threshold summary jq -n --arg uid "$1" --arg title "$2" --arg for "$3" --arg expr "$4" \ --arg op "$5" --argjson th "$6" --arg summary "$7" '{ uid: $uid, title: $title, condition: "C", for: $for, noDataState: "OK", execErrState: "Error", annotations: {summary: $summary}, data: [ {refId: "A", datasourceUid: "prometheus", relativeTimeRange: {from: 600, to: 0}, model: {refId: "A", expr: $expr, instant: true}}, {refId: "C", datasourceUid: "__expr__", model: {refId: "C", type: "threshold", expression: "A", conditions: [{evaluator: {type: $op, params: [$th]}}]}} ]}' } rules=$(jq -s . </dev/null echo "Alerts ready: 5 rules in 'VM alerts', sent to $OWNER by email."