Terraform helm provider with Azure firewall

Question

I'm using helm provider to install redis helm chart. The terraform configuration is given below.

provider "helm" {
  kubernetes {
   host = azurerm_kubernetes_cluster.aks.kube_config.0.host
   cluster_ca_certificate = 
base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)

  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    args = [
      "get-token",
      "--environment", "AzurePublicCloud",
      "--server-id", "6dae42f8-4368-4678-94ff-3960e28e3630", # The AAD server app ID of 
AKS Managed AAD is always 6dae42f8-4368-4678-94ff-3960e28e3630 in any environments.
    "--client-id", 
"${yamldecode(azurerm_kubernetes_cluster.aks.kube_config_raw).users[0].user.auth-provider.config.client-id}", 
    "--tenant-id", data.azurerm_client_config.current.tenant_id,
    "--login", "devicecode"
  ]
  command = "kubelogin"
  }
}
}

  resource "helm_release" "redis" {
  name             = "redis"
  repository       = "https://charts.bitnami.com/bitnami"
  chart            = "redis"
  version          = "16.11.2"
 namespace        = "redis-app"
 create_namespace = true


  set {
    name  = "cluster.enabled"
    value = "true"
  }

 }

I could see the helm chart is deploying in the cluster and helm release got created. But inisde the redis-app namespace the pods failed and in the pod logs I could see following error.

► checking prerequisites
✗ Kubernetes API call failed: Get "https://10.0.0.3:443/version": EOF

With further troubleshooting we identified that the issue could be because of the Azure Firewall dropping the traffic. In the above API URL if we could replace the 10.0.0.3 with the FQDN hostname (clustername-c2122e83.hcp.eastus.azmk8s.io) then it will allow it

How I could resolve this issue

Answer 1

An error indicates the Kubernetes API server is detected as a Yaml 10.0.0.3 address [Private IP], which will be inaccessible from within the cluster. If errors come out like this, it means that KUBECONFIG is not set in my view.

When we lookup the aks dns host which interacts with public ip configuration

if we browse:

looks like it's an issue with setting up KUBECONFIG. refer tutorial.

Here is the sample code base of replication

 resource "azurerm_resource_group" "example" {
  name     = "**********"
  location = "West Europe"
}

resource "azurerm_kubernetes_cluster" "aks" {
  name                = "exampleaks"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name
  dns_prefix          = "exampleaks"

  default_node_pool {
    name       = "default"
    node_count = 1
    vm_size    = "Standard_D2_v2"
  }

  identity {
    type = "SystemAssigned"
  }

  tags = {
    Environment = "devtest"
  }
}

output "client_certificate" {
  value     = azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate
  sensitive = true
}

output "kube_config" {
  value = azurerm_kubernetes_cluster.aks.kube_config_raw

  sensitive = true
}

data "azuread_client_config" "current" {}
provider "helm" {
  kubernetes {
   host = azurerm_kubernetes_cluster.aks.kube_config.0.host
   cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    args = [
      "get-token",
      "--environment", "AzurePublicCloud",
      "--server-id", "***********************", # The AAD server app ID of AKS Managed AAD is always ***************** in any environments.
    "--client-id", "**********-***********
    -tenant-id","*****************",
    "--login", "devicecode"
  ]
  command = "kubelogin"
  }
}
}

  resource "helm_release" "redis" {
  name             = "redis"
  repository       = "https://charts.bitnami.com/bitnami"
  chart            = "redis"
  version          = "16.11.2"
 namespace        = "redis-app"
 create_namespace = true
  set {
    name  = "cluster.enabled"
    value = "true"
  }

 }

Upon running of plan and apply