Unable to access one container from another container via Public IP/Domain

Question

I have two docker containers (zitadel and oauth2-proxy) running on the same VPS host machine with traefik acting as the reverse proxy for public access to the containers.

There is a need where oauth2-proxy container has to communicate with the zitadel container using the public domain name (Ex: zitadel.example.com) for OIDC provider verification. But oauth2-proxy is unable to communicate via public domain name and returns a TCP connection timeout error.

Note: All the containers are running on the same docker network.

Below are the respective docker-compose files for your reference:

zitadel docker-compose file:

version: "3.8"

services:
  zitadel:
    container_name: "zitadel"
    hostname: zitadel
    restart: "unless-stopped"
    networks:
      - "web"
    image: "ghcr.io/zitadel/zitadel:stable"
    command: 'start-from-init --config /example-zitadel-config.yaml --config /example-zitadel-secrets.yaml --steps /example-zitadel-init-steps.yaml --masterkey "nionib" --tlsMode external'
    environment:
      ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME: Admin1!@zitadel
      ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD: 
    volumes:
      - "./example-zitadel-config.yaml:/example-zitadel-config.yaml:ro"
      - "./example-zitadel-secrets.yaml:/example-zitadel-secrets.yaml:ro"
      - "./example-zitadel-init-steps.yaml:/example-zitadel-init-steps.yaml:ro"
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.zitadel.entrypoints=http"
      - "traefik.http.routers.zitadel.rule=Host(`zitadel.example.com`)"
      - "traefik.http.middlewares.zitadel-https-redirect.redirectscheme.scheme=https"
      - "traefik.http.routers.zitadel.middlewares=zitadel-https-redirect"
      - "traefik.http.routers.zitadel-secure.entrypoints=https"
      - "traefik.http.routers.zitadel-secure.rule=Host(`zitadel.example.com`)"
      - "traefik.http.routers.zitadel-secure.service=zitadel"
      - "traefik.http.routers.zitadel-secure.tls=true"
      - "traefik.http.routers.zitadel-secure.tls.certresolver=wildcard"
      - "traefik.http.routers.zitadel-secure.tls.domains[0].main=example.com"
      - "traefik.http.routers.zitadel-secure.tls.domains[0].sans=*.example.com"
      - "traefik.http.services.zitadel.loadbalancer.server.scheme=h2c"
      - "traefik.http.services.zitadel.loadbalancer.passHostHeader=true"
      - "traefik.http.services.zitadel.loadbalancer.server.port=8080"
      - "traefik.docker.network=web"

networks:
  web:
    external: true

oauth2 docker-compose file

version: '3.9'

services:
  oauth2:
    container_name: "oauth2"
    hostname: oauth2
    image: quay.io/oauth2-proxy/oauth2-proxy:latest
    networks:
      - web
    command:
      - --http-address=0.0.0.0:4180
      - --https-address=0.0.0.0:4181
      - --cookie-secret=${COOKIE_SECRET}
      - --cookie-secure=${COOKIE_SECURE}
      - --email-domain=${EMAIL_DOMAIN}
      - --provider=${PROVIDER}
      - --provider-display-name=${PROVIDER_DISPLAY_NAME}
      - --client-id=${CLIENT_ID}
      - --client-secret=${CLIENT_SECRET}
      - --oidc-issuer-url=${OIDE_ISSUER_URL}
      - --oidc-groups-claim=${OIDC_GROUPS_CLAIM}
      - --pass-access-token=true
      - --skip-provider-button=true
      - --redirect-url=${REDIRECT_URL}
    healthcheck:
      test: ['CMD', 'wget', '--spider', 'http://localhost:4180/ping']
    labels:
      - traefik.enable=true
      - traefik.http.services.oauth2-proxy.loadbalancer.server.port=4180
      - traefik.http.services.oauth2-proxy.loadbalancer.passHostHeader=true
      - traefik.http.routers.oauth2-proxy.rule=PathPrefix(`/oauth2/`)
      - traefik.http.routers.oauth2-proxy.priority=2147483647
      - traefik.http.routers.oauth2-proxy.tls=true
      - traefik.http.routers.oauth2-proxy.tls.certresolver=wildcard
      - traefik.http.routers.oauth2-proxy.tls.domains[0].main=example.com
      - traefik.http.routers.oauth2-proxy.tls.domains[0].sans=*.example.com
      - traefik.docker.network=web
      - traefik.http.middlewares.auth-errors.errors.status=401-403
      - traefik.http.middlewares.auth-errors.errors.service=oauth2@docker
      - traefik.http.middlewares.auth-errors.errors.query=/oauth2/sign_in

networks:
  web:
    external: true

traefik docker-compose file:

version: '3'

services:
  traefik:
    image: traefik:v3.0
    container_name: traefik
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    networks:
      - web
    ports:
      - "80:80"             # HTTP
      - "443:443"           # HTTPS
    env_file:
      - ./.env
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./data/traefik.yml:/traefik.yml:ro
      - ./data:/data
      - ./data/routes.yml:/routes.yml
      - ./logs/:/logs/
    command:
      - "--accesslog=true"
      - "--accesslog.filePath=/logs/access.log"
      - '--api=true'
      - '--api.dashboard=true'
      - '--api.insecure=false'
      - '--pilot.dashboard=false'
      - '--global.sendAnonymousUsage=false'
      - '--global.checkNewVersion=true'
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik.entrypoints=http"
      - "traefik.http.routers.traefik.rule=Host(`example.com`)"
      - "traefik.http.middlewares.traefik-https-redirect.redirectscheme.scheme=https"
      - "traefik.http.routers.traefik.middlewares=traefik-https-redirect"
      - "traefik.http.routers.traefik-secure.entrypoints=https"
      - "traefik.http.routers.traefik-secure.rule=Host(`example.com`)"
      - "traefik.http.routers.traefik-secure.middlewares=authelia@docker"
      - "traefik.http.routers.traefik-secure.tls=true"
      - "traefik.http.routers.traefik-secure.tls.certresolver=https"
      - "traefik.http.routers.traefik-secure.service=api@internal"

networks:
  web:
    external: true

Traefik config:

global:
  checkNewVersion: true
  sendAnonymousUsage: false

api:
  dashboard: true

entryPoints:
  http:
    address: ":80"
    http:
      redirections:
        entrypoint:
          to: https
          scheme: https

  https:
    address: ":443"

providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false

  file:
    # filename: "/routes.yml"
    directory: "/data/routes"
    watch: true

certificatesResolvers:
  http:
    acme:
      email: [email protected]
      storage: /data/acme.json
      httpChallenge:
        entryPoint: http
  https:
    acme:
      email: [email protected]
      storage: /data/acme.json
      httpChallenge:
        entryPoint: https
  wildcard:
    acme:
      email: [email protected]
      storage: /data/acme.json
      dnsChallenge:
        provider: googledomains

  test:
    acme:
      email: [email protected]
      storage: /data/acme.json
      dnsChallenge:
        provider: googledomains

# Writing Logs to a File, in JSON
log:
  filePath: "/logs/access.log"
  format: json
  level: WARN

accessLog:
  filePath: "/logs/accessLog.log"
  bufferingSize: 100
  format: json
  filters:
    statusCodes:
      - "400-499"
      - "500-599"
    retryAttempts: true
    minDuration: "2500ms"

I tried to access my OIDC URL endpoint from a browser and I get a response back. To cross verify, I tried accessing the same URL via wget on a separate ubuntu container which also did not work either.

To confirm that the problem is not with the zitadel container, I tried a public domain name such as https://stackoverflow.com as OIDC provider URL and it didn't return a timeout error, but a 404 error which suggests that there is absolutely no problem with the zitadel container.

I also tried to perform wget on zitadel.example.com on the VPS machine and it worked fine, which narrows down the problem to be specific to my docker network setup, or so I think.

I even tried adding the below extra host to the oauth2-proxy docker-compose file which did not work.

extra_hosts:
      - "host.docker.internal:host-gateway"

please let me know if you need any further information