How To Configure Alertmanager with Recievers in Helm Chart for Kube-Prometheus-Stack

271 views Asked by At

So I need to configure some receivers for alertmanager:

  • Email
  • OpsGenie
  • MS Teams Notification

My Terraform Module looks like this:

terraform {
  required_version = ">= 1.3.6"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "4.56.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "2.11.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.23.0"
    }
  }
}

# https://artifacthub.io/packages/helm/argo/argo-workflows
# TODO: MOVE TO https://dev.to/kaitoii11/deploy-prometheus-monitoring-stack-to-kubernetes-with-a-single-helm-chart-2fbd
resource "helm_release" "kube-prometheus-stack" {
  name       = "kube-prometheus-stack"
  repository = "https://prometheus-community.github.io/helm-charts"
  chart      = "kube-prometheus-stack"
  version    = "51.5.2"
  namespace  = var.chart_namespace
  lint       = true
  timeout    = "600"

  values = [var.values_file_contents]

  reuse_values     = true
  force_update     = true
  recreate_pods    = true
  cleanup_on_fail  = true
  create_namespace = true
  atomic           = true
}

variable "chart_namespace" {
  type        = string
  default     = "kube-system"
  description = "Namespace Variable for the namespace resources get deployed"
}

variable "values_file_contents" {
  description = "contents of the values file to deploy the helm chart with"
  type        = string
}

My values file looks like this:

grafana:
  ingress:
    enabled: "true"
    annotations:
      alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
      alb.ingress.kubernetes.io/healthcheck-path: /
      alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
      alb.ingress.kubernetes.io/wafv2-acl-arn: <some_wF_ARN>
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/ssl-redirect: "443"
      alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
      alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
      alb.ingress.kubernetes.io/target-type: ip
      external-dns.alpha.kubernetes.io/hostname: <GRAFANA_ADDRESS>
      kubernetes.io/ingress.class: alb
    hosts:
      - <GRAFANA_ADDRESS>
    paths:
      - /*
alertmanager:
#  alertmanagerSpec:
#    logLevel: "warning"
  config:
    route:
      group_by: [ 'job', 'alertname', 'priority','namespace' ]
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      receiver:
        - "email_receiver"
        - "ms_teams_receiver"
        - "opsgenie_receiver"
      routes:
        - match:
            alertname: Watchdog
            receiver: 'null'
        - receiver: "email_receiver"
        - receiver: "ms_teams_receiver"
        - receiver: "opsgenie_receiver"
    receivers:
      - name: 'null'
      - name: 'email_receiver'
        email_config:
        - to: "<SOME_EMAIL>"
          from: "prometheus@<HOSTNAME>"
          require_tls: false
          send_resolved: true
      - name: "ms_teams_receiver"
        msteams_config:
          webhook_url: "<SOME_MS_TEAMS_WEBHOOK>"
          send_resolved: true
      - name: "opsgenie_receiver"
        opsgenie_config:
          api_key: "<OPS_GENIE_APIKEY>"
          api_url: "<OPS_GENIE_APIURL"
          send_resolved: "true"
  ingress:
    enabled: "true"
    annotations:
      alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
      alb.ingress.kubernetes.io/healthcheck-path: /
      alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
      alb.ingress.kubernetes.io/wafv2-acl-arn: <SOME_WAF_ARN>
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/ssl-redirect: "443"
      alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
      alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
      alb.ingress.kubernetes.io/target-type: ip
      external-dns.alpha.kubernetes.io/hostname: <SOME_ADDRESS>
      kubernetes.io/ingress.class: alb
    hosts:
      - <SOME_ADDRESS>
    paths:
      - /*
prometheus:
  prometheusSpec:
#    logLevel: "warning"
    enableRemoteWriteReceiver: true
    remoteWrite:
      - url: "<SOME_ADDRESS>"
  ingress:
    enabled: "true"
    annotations:
      alb.ingress.kubernetes.io/certificate-arn: <SOME_CERTIFICATE>
      alb.ingress.kubernetes.io/healthcheck-path: /
      alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]'
      alb.ingress.kubernetes.io/wafv2-acl-arn: <SOME_ARN>
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/ssl-redirect: "443"
      alb.ingress.kubernetes.io/subnets: <SOME_SUBNET>
      alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
      alb.ingress.kubernetes.io/target-type: ip
      external-dns.alpha.kubernetes.io/hostname: <SOME_ADDRESS>
      kubernetes.io/ingress.class: alb
    hosts:
      - <SOME_ADDRESS>
    paths:
      - /*
#thanosRuler:
#  thanosRulerSpec:
#    logLevel: "warning"

The main part is when I configure alertmanager with the following, it doesn't actually set anything up:

alertmanager:
#  alertmanagerSpec:
#    logLevel: "warning"
  config:
    route:
      group_by: [ 'job', 'alertname', 'priority','namespace' ]
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      receiver:
        - "email_receiver"
        - "ms_teams_receiver"
        - "opsgenie_receiver"
      routes:
        - match:
            alertname: Watchdog
            receiver: 'null'
        - receiver: "email_receiver"
        - receiver: "ms_teams_receiver"
        - receiver: "opsgenie_receiver"
    receivers:
      - name: 'null'
      - name: 'email_receiver'
        email_config:
        - to: "<SOME_EMAIL>"
          from: "prometheus@<HOSTNAME>"
          require_tls: false
          send_resolved: true
      - name: "ms_teams_receiver"
        msteams_config:
          webhook_url: "<SOME_MS_TEAMS_WEBHOOK>"
          send_resolved: true
      - name: "opsgenie_receiver"
        opsgenie_config:
          api_key: "<OPS_GENIE_APIKEY>"
          api_url: "<OPS_GENIE_APIURL"
          send_resolved: "true"

I have tried playing around with both the routes and the recievers, but when I check the alertmanager status page I don't really see anything configured, instead it just says null. I also, tried to do a template update to see if it would respect those values and pipe it to a file for inspection, and basically saw that they were not configured. This is the first time I have configured prometheus alertmanager receivers, so any help would be greatly appreciated.

0

There are 0 answers