Error when referring output of one terraform module into another module

Question

I am new to terraform and trying to create a project in terraform v0.14.0 that will perform following steps:

1. Create a n number of ec2 machines (one will be master and rest will be child)
2. Create a NLB and terget group
3. Attach child ec2 instance private ip to target group by referencing a output variable of step 1 (ec2 module).

But when I am running terraform plan I am getting below message. Can this be fixed? or any other approach to provision these resources in one go?

Error: Invalid for_each argument

  on ../../../../modules/nlb_new2/main.tf line 493, in resource "aws_lb_target_group_attachment" "tgr_attachment":
 493:   for_each = {
 494:           for pair in setproduct(keys(tomap({arn = lookup(aws_lb_target_group.main[0],"arn")})),  **var.target_id**) :
 495:           "${pair[0]},${pair[1]}" => {
 496:             target_group = aws_lb_target_group.main[0].arn
 497:             target       = length(split(":",pair[1])) > 0 ? split(":",pair[1])[0]: null
 498:             port         = length(split(":",pair[1])) == 2 ? split(":",pair[1])[1] : null
 499:           }
 500:     }

#project level main.tf

  region = lookup(var.lb_common_prop, "region", "us-east-1")
}

module "ec2_detail" {
  source = "../../../../modules/ec2_new2"

  for_each = var.ec2_detail

  instance_count = each.value.instance_count

  name =  format("%s-${var.ec2_name.suffix}",each.key)
  ami_name      = var.ec2_common_prop.ami_name
  instance_type = var.ec2_common_prop.instance_type
  subnet_id     = ""
  vpc_name      = var.ec2_common_prop.vpc_name
  
  subnet_names  = var.security.subnet_names
  vpc_security_group_names    = var.security.vpc_security_group_names
  
  associate_public_ip_address = var.ec2_common_prop.associate_public_ip_address
  tags                        = var.ec2_tags
  volume_tags                 = var.ec2_volume_tags
  key_name                    = var.ec2_common_prop.key_name
  user_data                   = ""
  iam_instance_profile        = var.ec2_common_prop.iam_instance_profile
  monitoring                  = var.ec2_common_prop.monitoring
  source_dest_check           = var.ec2_common_prop.source_dest_check
  placement_group             = var.ec2_common_prop.placement_group
  ebs_optimized               = var.ec2_common_prop.ebs_optimized
  disable_api_termination     = var.ec2_common_prop.disable_api_termination
  root_block_device = var.hardware.root_block_device
  ebs_block_device  = var.hardware.ebs_block_device

}


module "network_load_balancer" {
  depends_on=[module.ec2_detail]
  source = "../../../../modules/nlb_new2"
  
  for_each = var.all_load_balancer
  name = format("%s-${var.ec2_name.suffix}",each.key)

  load_balancer_type = each.value.load_balancer_type

  vpc_name      = var.ec2_common_prop.vpc_name
  subnet_names  = var.security.subnet_names
  internal      = lookup(merge(var.lb_tags, each.value.lb_tags), "internal", true)
   
  target_id = lookup(each.value.target_groups[0],"target_id",[])==[] ? [for pair in setproduct(module.ec2_detail["memsql-child"].private_ip, [lookup(var.lb_common_prop,"target_port")]) :  "${pair[0]}:${pair[1]}" ] : lookup(each.value.target_groups[0],"target_id",[])
  
  target_groups = each.value.target_groups
  http_tcp_listeners = each.value.http_tcp_listeners
  tags  = merge(var.lb_tags, each.value.lb_tags)
}

---

#NLB module main.tf

data aws_vpc "selected" {
    filter {
    name   = "tag:Name"
    values = [var.vpc_name] 
  }
}

data "aws_subnet_ids" "selected" {
  vpc_id=data.aws_vpc.selected.id
  filter {
    name   = "tag:Name"
    values = var.subnet_names
  }
}

#data "aws_subnet" "selected" {
#  for_each = data.aws_subnet_ids.selected.ids
#  id       = each.value
#}

resource "aws_lb" "this" {
  count = var.create_lb ? 1 : 0

  name        = var.name
  name_prefix = var.name_prefix

  load_balancer_type = var.load_balancer_type
  internal           = var.internal
  #security_groups    = var.security_groups
  
  #security_groups    = [for i in range(length(var.vpc_security_group_names)) : data.aws_security_group.selected.*.id[i]]
  
  #subnets            = var.subnets
  #subnets = [for s in data.aws_subnet_ids.selected : s.ids]
  subnets = data.aws_subnet_ids.selected.ids

  idle_timeout                     = var.idle_timeout
  enable_cross_zone_load_balancing = var.enable_cross_zone_load_balancing
  enable_deletion_protection       = var.enable_deletion_protection
  enable_http2                     = var.enable_http2
  ip_address_type                  = var.ip_address_type
  drop_invalid_header_fields       = var.drop_invalid_header_fields

  # See notes in README (ref: https://github.com/terraform-providers/terraform-provider-aws/issues/7987)
  dynamic "access_logs" {
    for_each = length(keys(var.access_logs)) == 0 ? [] : [var.access_logs]

    content {
      enabled = lookup(access_logs.value, "enabled", lookup(access_logs.value, "bucket", null) != null)
      bucket  = lookup(access_logs.value, "bucket", null)
      prefix  = lookup(access_logs.value, "prefix", null)
    }
  }

  dynamic "subnet_mapping" {
    for_each = var.subnet_mapping

    content {
      subnet_id     = subnet_mapping.value.subnet_id
      allocation_id = lookup(subnet_mapping.value, "allocation_id", null)
    }
  }

  tags = merge(
    var.tags,
    var.lb_tags,
    {
      Name = var.name != null ? var.name : var.name_prefix
      "name" =  var.name != null ? var.name : var.name_prefix
    },
  )

  timeouts {
    create = var.load_balancer_create_timeout
    update = var.load_balancer_update_timeout
    delete = var.load_balancer_delete_timeout
  }
}

resource "aws_lb_target_group" "main" {
  count = var.create_lb ? length(var.target_groups) : 0

  name        = lookup(var.target_groups[count.index], "name", null)
  name_prefix = lookup(var.target_groups[count.index], "name_prefix", null)
  
  #vpc_id      = var.vpc_id
  vpc_id      = data.aws_vpc.selected.id
  port        = lookup(var.target_groups[count.index], "backend_port", null)
  protocol    = lookup(var.target_groups[count.index], "backend_protocol", null) != null ? upper(lookup(var.target_groups[count.index], "backend_protocol")) : null
  target_type = lookup(var.target_groups[count.index], "target_type", null)

  deregistration_delay               = lookup(var.target_groups[count.index], "deregistration_delay", null)
  slow_start                         = lookup(var.target_groups[count.index], "slow_start", null)
  proxy_protocol_v2                  = lookup(var.target_groups[count.index], "proxy_protocol_v2", false)
  lambda_multi_value_headers_enabled = lookup(var.target_groups[count.index], "lambda_multi_value_headers_enabled", false)
  load_balancing_algorithm_type      = lookup(var.target_groups[count.index], "load_balancing_algorithm_type", null)

  dynamic "health_check" {
    for_each = length(keys(lookup(var.target_groups[count.index], "health_check", {}))) == 0 ? [] : [lookup(var.target_groups[count.index], "health_check", {})]

    content {
      enabled             = lookup(health_check.value, "enabled", null)
      interval            = lookup(health_check.value, "interval", null)
      path                = lookup(health_check.value, "path", null)
      port                = lookup(health_check.value, "port", null)
      healthy_threshold   = lookup(health_check.value, "healthy_threshold", null)
      unhealthy_threshold = lookup(health_check.value, "unhealthy_threshold", null)
      timeout             = lookup(health_check.value, "timeout", null)
      protocol            = lookup(health_check.value, "protocol", null)
      matcher             = lookup(health_check.value, "matcher", null)
    }
  }

  dynamic "stickiness" {
    for_each = length(keys(lookup(var.target_groups[count.index], "stickiness", {}))) == 0 ? [] : [lookup(var.target_groups[count.index], "stickiness", {})]

    content {
      enabled         = lookup(stickiness.value, "enabled", null)
      cookie_duration = lookup(stickiness.value, "cookie_duration", null)
      type            = lookup(stickiness.value, "type", null)
    }
  }

  tags = merge(
    var.tags,
    {"stack-technology"       = "target-group"},
    var.target_group_tags,
    #lookup(var.target_groups[count.index], "tags", {}),
    {
      "Name" = format("%s${var.name}", lookup(var.target_groups[count.index], "name_prefix", ""))
      "name" =  format("%s${var.name}", lookup(var.target_groups[count.index], "name_prefix", "")) 
    },
  )

  depends_on = [aws_lb.this]

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_lb_listener_rule" "https_listener_rule" {
  count = var.create_lb ? length(var.https_listener_rules) : 0

  listener_arn = aws_lb_listener.frontend_https[lookup(var.https_listener_rules[count.index], "https_listener_index", count.index)].arn
  priority     = lookup(var.https_listener_rules[count.index], "priority", null)

  # authenticate-cognito actions
  dynamic "action" {
    for_each = [
      for action_rule in var.https_listener_rules[count.index].actions :
      action_rule
      if action_rule.type == "authenticate-cognito"
    ]

    content {
      type = action.value["type"]
      authenticate_cognito {
        authentication_request_extra_params = lookup(action.value, "authentication_request_extra_params", null)
        on_unauthenticated_request          = lookup(action.value, "on_authenticated_request", null)
        scope                               = lookup(action.value, "scope", null)
        session_cookie_name                 = lookup(action.value, "session_cookie_name", null)
        session_timeout                     = lookup(action.value, "session_timeout", null)
        user_pool_arn                       = action.value["user_pool_arn"]
        user_pool_client_id                 = action.value["user_pool_client_id"]
        user_pool_domain                    = action.value["user_pool_domain"]
      }
    }
  }

  # authenticate-oidc actions
  dynamic "action" {
    for_each = [
      for action_rule in var.https_listener_rules[count.index].actions :
      action_rule
      if action_rule.type == "authenticate-oidc"
    ]

    content {
      type = action.value["type"]
      authenticate_oidc {
        # Max 10 extra params
        authentication_request_extra_params = lookup(action.value, "authentication_request_extra_params", null)
        authorization_endpoint              = action.value["authorization_endpoint"]
        client_id                           = action.value["client_id"]
        client_secret                       = action.value["client_secret"]
        issuer                              = action.value["issuer"]
        on_unauthenticated_request          = lookup(action.value, "on_unauthenticated_request", null)
        scope                               = lookup(action.value, "scope", null)
        session_cookie_name                 = lookup(action.value, "session_cookie_name", null)
        session_timeout                     = lookup(action.value, "session_timeout", null)
        token_endpoint                      = action.value["token_endpoint"]
        user_info_endpoint                  = action.value["user_info_endpoint"]
      }
    }
  }

  # redirect actions
  dynamic "action" {
    for_each = [
      for action_rule in var.https_listener_rules[count.index].actions :
      action_rule
      if action_rule.type == "redirect"
    ]

    content {
      type = action.value["type"]
      redirect {
        host        = lookup(action.value, "host", null)
        path        = lookup(action.value, "path", null)
        port        = lookup(action.value, "port", null)
        protocol    = lookup(action.value, "protocol", null)
        query       = lookup(action.value, "query", null)
        status_code = action.value["status_code"]
      }
    }
  }

  # fixed-response actions
  dynamic "action" {
    for_each = [
      for action_rule in var.https_listener_rules[count.index].actions :
      action_rule
      if action_rule.type == "fixed-response"
    ]

    content {
      type = action.value["type"]
      fixed_response {
        message_body = lookup(action.value, "message_body", null)
        status_code  = lookup(action.value, "status_code", null)
        content_type = action.value["content_type"]
      }
    }
  }

  # forward actions
  dynamic "action" {
    for_each = [
      for action_rule in var.https_listener_rules[count.index].actions :
      action_rule
      if action_rule.type == "forward"
    ]

    content {
      type             = action.value["type"]
      target_group_arn = aws_lb_target_group.main[lookup(action.value, "target_group_index", count.index)].id
    }
  }

  # Path Pattern condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "path_patterns", [])) > 0
    ]

    content {
      path_pattern {
        values = condition.value["path_patterns"]
      }
    }
  }

  # Host header condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "host_headers", [])) > 0
    ]

    content {
      host_header {
        values = condition.value["host_headers"]
      }
    }
  }

  # Http header condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "http_headers", [])) > 0
    ]

    content {
      dynamic "http_header" {
        for_each = condition.value["http_headers"]

        content {
          http_header_name = http_header.value["http_header_name"]
          values           = http_header.value["values"]
        }
      }
    }
  }

  # Http request method condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "http_request_methods", [])) > 0
    ]

    content {
      http_request_method {
        values = condition.value["http_request_methods"]
      }
    }
  }

  # Query string condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "query_strings", [])) > 0
    ]

    content {
      dynamic "query_string" {
        for_each = condition.value["query_strings"]

        content {
          key   = lookup(query_string.value, "key", null)
          value = query_string.value["value"]
        }
      }
    }
  }

  # Source IP address condition
  dynamic "condition" {
    for_each = [
      for condition_rule in var.https_listener_rules[count.index].conditions :
      condition_rule
      if length(lookup(condition_rule, "source_ips", [])) > 0
    ]

    content {
      source_ip {
        values = condition.value["source_ips"]
      }
    }
  }
}


resource "aws_lb_listener" "frontend_http_tcp" {
  count = var.create_lb ? length(var.http_tcp_listeners) : 0

  load_balancer_arn = aws_lb.this[0].arn

  port     = var.http_tcp_listeners[count.index]["port"]
  protocol = var.http_tcp_listeners[count.index]["protocol"]

  dynamic "default_action" {
    for_each = length(keys(var.http_tcp_listeners[count.index])) == 0 ? [] : [var.http_tcp_listeners[count.index]]

    # Defaults to forward action if action_type not specified
    content {
      type             = lookup(default_action.value, "action_type", "forward")
      target_group_arn = contains([null, "", "forward"], lookup(default_action.value, "action_type", "")) ? aws_lb_target_group.main[lookup(default_action.value, "target_group_index", count.index)].id : null

      dynamic "redirect" {
        for_each = length(keys(lookup(default_action.value, "redirect", {}))) == 0 ? [] : [lookup(default_action.value, "redirect", {})]

        content {
          path        = lookup(redirect.value, "path", null)
          host        = lookup(redirect.value, "host", null)
          port        = lookup(redirect.value, "port", null)
          protocol    = lookup(redirect.value, "protocol", null)
          query       = lookup(redirect.value, "query", null)
          status_code = redirect.value["status_code"]
        }
      }

      dynamic "fixed_response" {
        for_each = length(keys(lookup(default_action.value, "fixed_response", {}))) == 0 ? [] : [lookup(default_action.value, "fixed_response", {})]

        content {
          content_type = fixed_response.value["content_type"]
          message_body = lookup(fixed_response.value, "message_body", null)
          status_code  = lookup(fixed_response.value, "status_code", null)
        }
      }
    }
  }
}

resource "aws_lb_listener" "frontend_https" {
  count = var.create_lb ? length(var.https_listeners) : 0

  load_balancer_arn = aws_lb.this[0].arn

  port            = var.https_listeners[count.index]["port"]
  protocol        = lookup(var.https_listeners[count.index], "protocol", "HTTPS")
  certificate_arn = var.https_listeners[count.index]["certificate_arn"]
  ssl_policy      = lookup(var.https_listeners[count.index], "ssl_policy", var.listener_ssl_policy_default)

  dynamic "default_action" {
    for_each = length(keys(var.https_listeners[count.index])) == 0 ? [] : [var.https_listeners[count.index]]

    # Defaults to forward action if action_type not specified
    content {
      type             = lookup(default_action.value, "action_type", "forward")
      target_group_arn = contains([null, "", "forward"], lookup(default_action.value, "action_type", "")) ? aws_lb_target_group.main[lookup(default_action.value, "target_group_index", count.index)].id : null

      dynamic "redirect" {
        for_each = length(keys(lookup(default_action.value, "redirect", {}))) == 0 ? [] : [lookup(default_action.value, "redirect", {})]

        content {
          path        = lookup(redirect.value, "path", null)
          host        = lookup(redirect.value, "host", null)
          port        = lookup(redirect.value, "port", null)
          protocol    = lookup(redirect.value, "protocol", null)
          query       = lookup(redirect.value, "query", null)
          status_code = redirect.value["status_code"]
        }
      }

      dynamic "fixed_response" {
        for_each = length(keys(lookup(default_action.value, "fixed_response", {}))) == 0 ? [] : [lookup(default_action.value, "fixed_response", {})]

        content {
          content_type = fixed_response.value["content_type"]
          message_body = lookup(fixed_response.value, "message_body", null)
          status_code  = lookup(fixed_response.value, "status_code", null)
        }
      }

      # Authentication actions only available with HTTPS listeners
      dynamic "authenticate_cognito" {
        for_each = length(keys(lookup(default_action.value, "authenticate_cognito", {}))) == 0 ? [] : [lookup(default_action.value, "authenticate_cognito", {})]

        content {
          # Max 10 extra params
          authentication_request_extra_params = lookup(authenticate_cognito.value, "authentication_request_extra_params", null)
          on_unauthenticated_request          = lookup(authenticate_cognito.value, "on_authenticated_request", null)
          scope                               = lookup(authenticate_cognito.value, "scope", null)
          session_cookie_name                 = lookup(authenticate_cognito.value, "session_cookie_name", null)
          session_timeout                     = lookup(authenticate_cognito.value, "session_timeout", null)
          user_pool_arn                       = authenticate_cognito.value["user_pool_arn"]
          user_pool_client_id                 = authenticate_cognito.value["user_pool_client_id"]
          user_pool_domain                    = authenticate_cognito.value["user_pool_domain"]
        }
      }

      dynamic "authenticate_oidc" {
        for_each = length(keys(lookup(default_action.value, "authenticate_oidc", {}))) == 0 ? [] : [lookup(default_action.value, "authenticate_oidc", {})]

        content {
          # Max 10 extra params
          authentication_request_extra_params = lookup(authenticate_oidc.value, "authentication_request_extra_params", null)
          authorization_endpoint              = authenticate_oidc.value["authorization_endpoint"]
          client_id                           = authenticate_oidc.value["client_id"]
          client_secret                       = authenticate_oidc.value["client_secret"]
          issuer                              = authenticate_oidc.value["issuer"]
          on_unauthenticated_request          = lookup(authenticate_oidc.value, "on_unauthenticated_request", null)
          scope                               = lookup(authenticate_oidc.value, "scope", null)
          session_cookie_name                 = lookup(authenticate_oidc.value, "session_cookie_name", null)
          session_timeout                     = lookup(authenticate_oidc.value, "session_timeout", null)
          token_endpoint                      = authenticate_oidc.value["token_endpoint"]
          user_info_endpoint                  = authenticate_oidc.value["user_info_endpoint"]
        }
      }
    }
  }

  dynamic "default_action" {
    for_each = contains(["authenticate-oidc", "authenticate-cognito"], lookup(var.https_listeners[count.index], "action_type", {})) ? [var.https_listeners[count.index]] : []
    content {
      type             = "forward"
      target_group_arn = aws_lb_target_group.main[lookup(default_action.value, "target_group_index", count.index)].id
    }
  }
}

resource "aws_lb_listener_certificate" "https_listener" {
  count = var.create_lb ? length(var.extra_ssl_certs) : 0

  listener_arn    = aws_lb_listener.frontend_https[var.extra_ssl_certs[count.index]["https_listener_index"]].arn
  certificate_arn = var.extra_ssl_certs[count.index]["certificate_arn"]
}


resource "aws_lb_target_group_attachment" "tgr_attachment" {
    depends_on = [aws_lb.this]
    for_each = {
        for pair in setproduct(keys(tomap({arn = lookup(aws_lb_target_group.main[0],"arn")})),  var.target_id) :
        "${pair[0]},${pair[1]}" => {
          target_group = aws_lb_target_group.main[0].arn
          target       = length(split(":",pair[1])) > 0 ? split(":",pair[1])[0]: null
          port         = length(split(":",pair[1])) == 2 ? split(":",pair[1])[1] : null
        }
      }
      
    target_group_arn = var.create_lb ? each.value.target_group : null
    target_id        = var.create_lb ? each.value.target : null
    port             = var.create_lb ? each.value.port : null
        
}

Answer 1

Your error message and your code examples don't seem to be complete, but in spite of that I think I can see what's causing the problem here: you're trying to use the target group ARNs as part of the instance keys for aws_lb_target_group_attachment.tgr_attachment, but the ARNs are not appropriate to use for instance keys because the provider doesn't know the ARN value until after each object is created, and so it can't predict the value during planning.

Instead, I'd suggest to change your var.target_groups to be a map of objects instead of a list of objects, and then use for_each = var.target_groups in aws_lb_target_group.main. The caller of your module can therefore specify a meaningful name for each of the target groups, which you can use as part of the instance keys for aws_lb_target_group_attachment.tgr_attachment:

resource "aws_lb_target_group_attachment" "tgr_attachment" {
  for_each = {
    for pair in setproduct(keys(aws_lb_target_group.main), var.target_id) :
    "${pair[0]},${pair[1]}" => {
      target_group_arn = aws_lb_target_group.main[pair[0]].arn
      target           = length(split(":",pair[1])) > 0 ? split(":",pair[1])[0]: null
      port             = length(split(":",pair[1])) == 2 ? split(":",pair[1])[1] : null
    }
  }

  # ...
}

By making the caller of the module specify a key for each of the target groups, you put the caller in control of whether a change to the collection of target groups should be understood as updating an existing target group in-place, creating a new target group, or removing an existing target group.

You also, as a nice side-effect, get a static and stable identifier to use for each of your instances which would be meaningful to someone reviewing the terraform plan output, and that will stay consistent even if a future change causes one of the target groups to be replaced.