AWS ECS Error: Task failed ELB health checks in target-group

146 views Asked by At

I am deploying a compose docker to AWS ECS. The compose contains two services: one postgres and our api running on port 8000.

$ compose-ecs up
ApiService ServiceSchedulerInitiated: Task failed ELB health checks in (target-group arn:aws:elasticloadbalancing:us-east-1:821470956622:targetgroup/laclie-ApiTC-HTNTTIBU9DVM/03d34a43d358c4b5)

compose.yaml

services:
  api:
    image: outsidethecode/la-client-api
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - "8000:8000" # Map API app port to host
    depends_on:
      - database # Make sure the API app starts after the database is ready
    environment:
      DATABASE_URL: "postgresql://postgres:postgres@database/laclient"
      ROCKET_ADDRESS: 0.0.0.0 
      ROCKET_ENV: release

  nginx:
    image: nginx:latest
    ports:
      - "80:80"
    depends_on:
      - api

  database:
    image: postgres:latest # Use the official PostgreSQL image
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
      POSTGRES_DB: laclient
    ports:
      - "5432:5432" # Map PostgreSQL port to host     

As to my understanding, to pass the health check we have to have a GET / resource to return 200. I've done that but that resource is on port 8000 similar to the rest of api resources.

Here is the convert version of the cloudformation:

AWSTemplateFormatVersion: 2010-09-09
Resources:
  ApiService:
    DependsOn:
    - ApiTCP8000Listener
    - DatabaseService
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      LoadBalancers:
      - ContainerName: api
        ContainerPort: 8000
        TargetGroupArn:
          Ref: ApiTCP8000TargetGroup
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - subnet-0bd97383cdf4d5896
          - subnet-0a52cf89f7c027b20
          - subnet-0ffafc731067a7d01
          - subnet-0de9ebed3a873fccf
          - subnet-00ae1e792af0e2e1d
          - subnet-0095a287059d86962
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - ApiServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: api
      TaskDefinition:
        Ref: ApiTaskDefinition
    Type: AWS::ECS::Service
  ApiServiceDiscoveryEntry:
    Properties:
      Description: '"api" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: api
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  ApiTCP8000Listener:
    Properties:
      DefaultActions:
      - ForwardConfig:
          TargetGroups:
          - TargetGroupArn:
              Ref: ApiTCP8000TargetGroup
        Type: forward
      LoadBalancerArn:
        Ref: LoadBalancer
      Port: 8000
      Protocol: TCP
    Type: AWS::ElasticLoadBalancingV2::Listener
  ApiTCP8000TargetGroup:
    Properties:
      Port: 8000
      Protocol: TCP
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      TargetType: ip
      VpcId: vpc-08cccbaf3a61d85bf
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
  ApiTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - laclientapi.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: Api_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Api_ResolvConf_InitContainer
        Environment:
        - Name: DATABASE_URL
          Value: postgresql://postgres:postgres@database/laclient
        - Name: ROCKET_ADDRESS
          Value: 0.0.0.0
        - Name: ROCKET_ENV
          Value: release
        Essential: true
        Image: docker.io/outsidethecode/la-client-api:latest@sha256:9768ba65e9a2d1aa5055960cf90003f8181ed99ca85043d494a762538789b161
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: api
        PortMappings:
        - ContainerPort: 8000
          HostPort: 8000
          Protocol: tcp
      Cpu: "256"
      ExecutionRoleArn:
        Ref: ApiTaskExecutionRole
      Family: laclientapi-api
      Memory: "512"
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
    Type: AWS::ECS::TaskDefinition
  ApiTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: api
    Type: AWS::IAM::Role
  CloudMap:
    Properties:
      Description: Service Map for Docker Compose project laclientapi
      Name: laclientapi.local
      Vpc: vpc-08cccbaf3a61d85bf
    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
  Cluster:
    Properties:
      ClusterName: laclientapi
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
    Type: AWS::ECS::Cluster
  DatabaseService:
    DependsOn:
    - DatabaseTCP5432Listener
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      LoadBalancers:
      - ContainerName: database
        ContainerPort: 5432
        TargetGroupArn:
          Ref: DatabaseTCP5432TargetGroup
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - subnet-0bd97383cdf4d5896
          - subnet-0a52cf89f7c027b20
          - subnet-0ffafc731067a7d01
          - subnet-0de9ebed3a873fccf
          - subnet-00ae1e792af0e2e1d
          - subnet-0095a287059d86962
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - DatabaseServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: database
      TaskDefinition:
        Ref: DatabaseTaskDefinition
    Type: AWS::ECS::Service
  DatabaseServiceDiscoveryEntry:
    Properties:
      Description: '"database" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: database
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  DatabaseTCP5432Listener:
    Properties:
      DefaultActions:
      - ForwardConfig:
          TargetGroups:
          - TargetGroupArn:
              Ref: DatabaseTCP5432TargetGroup
        Type: forward
      LoadBalancerArn:
        Ref: LoadBalancer
      Port: 5432
      Protocol: TCP
    Type: AWS::ElasticLoadBalancingV2::Listener
  DatabaseTCP5432TargetGroup:
    Properties:
      Port: 5432
      Protocol: TCP
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      TargetType: ip
      VpcId: vpc-08cccbaf3a61d85bf
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
  DatabaseTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - laclientapi.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: Database_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Database_ResolvConf_InitContainer
        Environment:
        - Name: POSTGRES_DB
          Value: laclient
        - Name: POSTGRES_PASSWORD
          Value: postgres
        - Name: POSTGRES_USER
          Value: postgres
        Essential: true
        Image: docker.io/library/postgres:latest@sha256:f1aaf6f8be5552bef66c5580efbd2942c37d7277cd0416ef4939fa34bf0baf31
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: database
        PortMappings:
        - ContainerPort: 5432
          HostPort: 5432
          Protocol: tcp
      Cpu: "256"
      ExecutionRoleArn:
        Ref: DatabaseTaskExecutionRole
      Family: laclientapi-database
      Memory: "512"
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
    Type: AWS::ECS::TaskDefinition
  DatabaseTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: database
    Type: AWS::IAM::Role
  Default8000Ingress:
    Properties:
      CidrIp: 0.0.0.0/0
      Description: api:8000/tcp on default network
      FromPort: 8000
      GroupId:
        Ref: DefaultNetwork
      IpProtocol: TCP
      ToPort: 8000
    Type: AWS::EC2::SecurityGroupIngress
  Default80Ingress:
    Properties:
      CidrIp: 0.0.0.0/0
      Description: nginx:80/tcp on default network
      FromPort: 80
      GroupId:
        Ref: DefaultNetwork
      IpProtocol: TCP
      ToPort: 80
    Type: AWS::EC2::SecurityGroupIngress
  Default5432Ingress:
    Properties:
      CidrIp: 0.0.0.0/0
      Description: database:5432/tcp on default network
      FromPort: 5432
      GroupId:
        Ref: DefaultNetwork
      IpProtocol: TCP
      ToPort: 5432
    Type: AWS::EC2::SecurityGroupIngress
  DefaultNetwork:
    Properties:
      GroupDescription: laclientapi Security Group for default network
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.network
        Value: laclientapi_default
      VpcId: vpc-08cccbaf3a61d85bf
    Type: AWS::EC2::SecurityGroup
  DefaultNetworkIngress:
    Properties:
      Description: Allow communication within network default
      GroupId:
        Ref: DefaultNetwork
      IpProtocol: "-1"
      SourceSecurityGroupId:
        Ref: DefaultNetwork
    Type: AWS::EC2::SecurityGroupIngress
  LoadBalancer:
    Properties:
      LoadBalancerAttributes:
      - Key: load_balancing.cross_zone.enabled
        Value: "true"
      Scheme: internet-facing
      Subnets:
      - subnet-0bd97383cdf4d5896
      - subnet-0a52cf89f7c027b20
      - subnet-0ffafc731067a7d01
      - subnet-0de9ebed3a873fccf
      - subnet-00ae1e792af0e2e1d
      - subnet-0095a287059d86962
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      Type: network
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
  LogGroup:
    Properties:
      LogGroupName: /docker-compose/laclientapi
    Type: AWS::Logs::LogGroup
  NginxService:
    DependsOn:
    - NginxTCP80Listener
    - ApiService
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      LoadBalancers:
      - ContainerName: nginx
        ContainerPort: 80
        TargetGroupArn:
          Ref: NginxTCP80TargetGroup
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - subnet-0bd97383cdf4d5896
          - subnet-0a52cf89f7c027b20
          - subnet-0ffafc731067a7d01
          - subnet-0de9ebed3a873fccf
          - subnet-00ae1e792af0e2e1d
          - subnet-0095a287059d86962
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - NginxServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: nginx
      TaskDefinition:
        Ref: NginxTaskDefinition
    Type: AWS::ECS::Service
  NginxServiceDiscoveryEntry:
    Properties:
      Description: '"nginx" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: nginx
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  NginxTCP80Listener:
    Properties:
      DefaultActions:
      - ForwardConfig:
          TargetGroups:
          - TargetGroupArn:
              Ref: NginxTCP80TargetGroup
        Type: forward
      LoadBalancerArn:
        Ref: LoadBalancer
      Port: 80
      Protocol: TCP
    Type: AWS::ElasticLoadBalancingV2::Listener
  NginxTCP80TargetGroup:
    Properties:
      Port: 80
      Protocol: TCP
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      TargetType: ip
      VpcId: vpc-08cccbaf3a61d85bf
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
  NginxTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - laclientapi.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: Nginx_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Nginx_ResolvConf_InitContainer
        Essential: true
        Image: docker.io/library/nginx:latest@sha256:32da30332506740a2f7c34d5dc70467b7f14ec67d912703568daff790ab3f755
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: laclientapi
        Name: nginx
        PortMappings:
        - ContainerPort: 80
          HostPort: 80
          Protocol: tcp
      Cpu: "256"
      ExecutionRoleArn:
        Ref: NginxTaskExecutionRole
      Family: laclientapi-nginx
      Memory: "512"
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
    Type: AWS::ECS::TaskDefinition
  NginxTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      Tags:
      - Key: com.docker.compose.project
        Value: laclientapi
      - Key: com.docker.compose.service
        Value: nginx
    Type: AWS::IAM::Role


I am not sure what to do to pass the ECS health check. Any help is much appreciated. Many thanks

0

There are 0 answers