How to create a docker composer environment where containers can communicate each other?

Question

My intention is to have:

• A kafka producer and exposes an API to send messages, running in a container
• A kafka consumer running in another container
• zookeeper running in another container
• kafka server running in another container

For the last two I use https://github.com/conduktor/kafka-stack-docker-compose?tab=readme-ov-file

I tried to add all of them to the same network using the bridge example but it didn't work. I keep getting from the producer messages on container log like

Id=adminclient-1] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
2024-03-30 08:37:30 2024-03-30T08:37:30.515Z  INFO 1 --- [| adminclient-1] org.apache.kafka.clients.NetworkClient   : [AdminClient clientId=adminclient-1] Node -1 disconnected.

Now I am trying the docker compose approach but I really don´t know where to start. What should be done about that so they all communicate each other?

version: '1.0'

services:
  zoo1:
    image: confluentinc/cp-zookeeper:7.3.2
    hostname: zoo1
    container_name: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888

  kafka1:
    image: confluentinc/cp-kafka:7.3.2
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    depends_on:
      - zoo1
  producer:
    image: producer:1.0
    hostname: producer1
    container_name: producer
    ports:
      - "8080:8080"
  consumer:
    image: consumer:1.0
    hostname: consumer1
    container_name: consumer
    ports:
      - "18080:18080"

My application.properties of the consumer project is like that

spring.kafka.producer.bootstrap-servers=localhost:9092
spring.kafka.consumer.group-id=groupid
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

Answer 1

The answer was not to as suggested.

The application.properties on consumer module was redefined to

#spring.kafka.consumer.bootstrap-servers=kafka:9092
spring.kafka.consumer.group-id=groupid
spring.kafka.bootstrap-servers=kafka:9092
product.kafkaServer= ${spring.kafka.bootstrap-servers}

spring.kafka.properties.security.protocol=PLAINTEXT
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

The application.properties on producer module was redefined to

spring.kafka.bootstrap-servers=kafka:9092
spring.kafka.producer.group-id=groupid

spring.kafka.properties.security.protocol=PLAINTEXT

spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

My Bean had to consume @Value("${spring.kafka.bootstrap-servers}")

Then the environment.yaml

version: '1.0'

services:

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 22181:2181

  kafka:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      #KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://kafka:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    depends_on:
      - kafka
    ports:
      - 8090:8080
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
  consumer:
    image: consumer:1.0.0
    hostname: consumer1
    container_name: consumer
    ports:
      - "8081:8081"
    depends_on:
      - kafka
  producer:
    image: producer:1.0.0
    hostname: producer1
    container_name: producer
    ports:
      - "8080:8080"
    depends_on:
      - consumer