So I need to configure some receivers for alertmanager:
- OpsGenie
- MS Teams Notification
My Terraform Module looks like this:
terraform {
required_version = ">= 1.3.6"
required_providers {
aws = {
source = "hashicorp/aws"
version = "4.56.0"
}
helm = {
source = "hashicorp/helm"
version = "2.11.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.23.0"
}
}
}
# https://artifacthub.io/packages/helm/argo/argo-workflows
# TODO: MOVE TO https://dev.to/kaitoii11/deploy-prometheus-monitoring-stack-to-kubernetes-with-a-single-helm-chart-2fbd
resource "helm_release" "kube-prometheus-stack" {
name = "kube-prometheus-stack"
repository = "https://prometheus-community.github.io/helm-charts"
chart = "kube-prometheus-stack"
version = "51.5.2"
namespace = var.chart_namespace
lint = true
timeout = "600"
values = [var.values_file_contents]
reuse_values = true
force_update = true
recreate_pods = true
cleanup_on_fail = true
create_namespace = true
atomic = true
}
variable "chart_namespace" {
type = string
default = "kube-system"
description = "Namespace Variable for the namespace resources get deployed"
}
variable "values_file_contents" {
description = "contents of the values file to deploy the helm chart with"
type = string
}
My values file looks like this:
grafana:
ingress:
enabled: "true"
annotations:
alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
alb.ingress.kubernetes.io/healthcheck-path: /
alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
alb.ingress.kubernetes.io/wafv2-acl-arn: <some_wF_ARN>
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/ssl-redirect: "443"
alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
alb.ingress.kubernetes.io/target-type: ip
external-dns.alpha.kubernetes.io/hostname: <GRAFANA_ADDRESS>
kubernetes.io/ingress.class: alb
hosts:
- <GRAFANA_ADDRESS>
paths:
- /*
alertmanager:
# alertmanagerSpec:
# logLevel: "warning"
config:
route:
group_by: [ 'job', 'alertname', 'priority','namespace' ]
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
receiver:
- "email_receiver"
- "ms_teams_receiver"
- "opsgenie_receiver"
routes:
- match:
alertname: Watchdog
receiver: 'null'
- receiver: "email_receiver"
- receiver: "ms_teams_receiver"
- receiver: "opsgenie_receiver"
receivers:
- name: 'null'
- name: 'email_receiver'
email_config:
- to: "<SOME_EMAIL>"
from: "prometheus@<HOSTNAME>"
require_tls: false
send_resolved: true
- name: "ms_teams_receiver"
msteams_config:
webhook_url: "<SOME_MS_TEAMS_WEBHOOK>"
send_resolved: true
- name: "opsgenie_receiver"
opsgenie_config:
api_key: "<OPS_GENIE_APIKEY>"
api_url: "<OPS_GENIE_APIURL"
send_resolved: "true"
ingress:
enabled: "true"
annotations:
alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
alb.ingress.kubernetes.io/healthcheck-path: /
alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
alb.ingress.kubernetes.io/wafv2-acl-arn: <SOME_WAF_ARN>
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/ssl-redirect: "443"
alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
alb.ingress.kubernetes.io/target-type: ip
external-dns.alpha.kubernetes.io/hostname: <SOME_ADDRESS>
kubernetes.io/ingress.class: alb
hosts:
- <SOME_ADDRESS>
paths:
- /*
prometheus:
prometheusSpec:
# logLevel: "warning"
enableRemoteWriteReceiver: true
remoteWrite:
- url: "<SOME_ADDRESS>"
ingress:
enabled: "true"
annotations:
alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
alb.ingress.kubernetes.io/healthcheck-path: /
alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
alb.ingress.kubernetes.io/wafv2-acl-arn: <SOME_ARN>
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/ssl-redirect: "443"
alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
alb.ingress.kubernetes.io/target-type: ip
external-dns.alpha.kubernetes.io/hostname: <SOME_ADDRESS>
kubernetes.io/ingress.class: alb
hosts:
- <SOME_ADDRESS>
paths:
- /*
#thanosRuler:
# thanosRulerSpec:
# logLevel: "warning"
The main part is when I configure alertmanager with the following, it doesn't actually set anything up:
alertmanager:
# alertmanagerSpec:
# logLevel: "warning"
config:
route:
group_by: [ 'job', 'alertname', 'priority','namespace' ]
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
receiver:
- "email_receiver"
- "ms_teams_receiver"
- "opsgenie_receiver"
routes:
- match:
alertname: Watchdog
receiver: 'null'
- receiver: "email_receiver"
- receiver: "ms_teams_receiver"
- receiver: "opsgenie_receiver"
receivers:
- name: 'null'
- name: 'email_receiver'
email_config:
- to: "<SOME_EMAIL>"
from: "prometheus@<HOSTNAME>"
require_tls: false
send_resolved: true
- name: "ms_teams_receiver"
msteams_config:
webhook_url: "<SOME_MS_TEAMS_WEBHOOK>"
send_resolved: true
- name: "opsgenie_receiver"
opsgenie_config:
api_key: "<OPS_GENIE_APIKEY>"
api_url: "<OPS_GENIE_APIURL"
send_resolved: "true"
I have tried playing around with both the routes and the recievers, but when I check the alertmanager status page I don't really see anything configured, instead it just says null. I also, tried to do a template update to see if it would respect those values and pipe it to a file for inspection, and basically saw that they were not configured. This is the first time I have configured prometheus alertmanager receivers, so any help would be greatly appreciated.