Nomad add the docker task to particular network

Question

I have a redpanda container which I want to use as a sink using vector when I am running this nomad job it is giving this error

Feb 27 08:57:01.931 ERROR rdkafka::client: librdkafka: Global error: Resolve (Local: Host resolution failure): redpanda-0:9092/bootstrap: Failed to resolve 'redpanda-0:9092': Name does not resolve (after 1ms in state CONNECT, 15 identical error(s) suppressed)    

this is my nomad job configuration

job "vector-redpanda" {
  datacenters = ["dc1"]
  # system job, runs on all nodes
  type = "system"
  update {
    min_healthy_time = "10s"
    healthy_deadline = "5m"
    progress_deadline = "10m"
    auto_revert = true
  }
  group "vector" {
    count = 1
    restart {
      attempts = 3
      interval = "10m"
      delay = "30s"
      mode = "fail"
    }
    network {
      port "api" {
        to = 8787
      }
      port "redpanda_network" {}
    }
    # docker socket volume
    volume "docker-sock-ro" {
      type = "host"
      source = "docker-sock-ro"
      read_only = true
    }
    ephemeral_disk {
      size    = 500
      sticky  = true
    }
    task "vector" {
      driver = "docker"
      config {
        image = "timberio/vector:0.14.X-alpine"
        ports = ["api", "redpanda_network"]
      }
      # docker socket volume mount
      volume_mount {
        volume = "docker-sock-ro"
        destination = "/var/run/docker.sock"
        read_only = true
      }
      # Vector won't start unless the sinks(backends) configured are healthy
      env {
        VECTOR_CONFIG = "local/vector.toml"
        VECTOR_REQUIRE_HEALTHY = "true"
      }
      # resource limits are a good idea because you don't want your log collection to consume all resources available
      resources {
        cpu    = 500 # 500 MHz
        memory = 256 # 256MB
      }
      # template with Vector's configuration
      template {
        destination = "local/vector.toml"
        change_mode   = "signal"
        change_signal = "SIGHUP"
        # overriding the delimiters to [[ ]] to avoid conflicts with Vector's native templating, which also uses {{ }}
        left_delimiter = "[["
        right_delimiter = "]]"
        data=<<EOH
                data_dir = "alloc/data/vector/"
              [api]
                enabled = true
                address = "0.0.0.0:8787"
                playground = true
              [sources.logs]
                type = "docker_logs"
              [transforms.records_for_redpanda]
                type = "lua"
                inputs = [ "logs" ]
                version = "2"
              [transforms.records_for_redpanda.hooks]
                process = """
                  function (event, emit)
                    event.log.job_id = event.log.label.job_id
                    event.log.task_id = event.log.label.task_id
                    event.log.run_id = event.log.label.run_id
                    event.log.created_at = event.log.timestamp
                    if event.log.run_id ~= "" then
                      emit(event)
                    end
                  end"""
              [sinks.my_kafka_sink]
                type = "kafka"
                inputs = [ "records_for_redpanda" ]
                bootstrap_servers = "redpanda-0:9092"
                topic = "logs_aggregation"
                acknowledgements.enabled = true
                compression = "gzip"
                healthcheck.enabled = false
                encoding.only_fields = ["job_id", "task_id", "run_id", "message", "created_at"]
                        encoding.timestamp_format = "unix"
                        encoding.codec = "json"
                message_timeout_ms = 450000
        EOH
      }
      service {
        check {
          port     = "api"
          type     = "http"
          path     = "/health"
          interval = "30s"
          timeout  = "5s"
        }
      }
      kill_timeout = "30s"
    }
  }
}    

I tried the same thing by creating a vector container in the same network as redpanda container network and it worked. I am not sure how I can do it for docker task in nomad.

Answer 1

You have to change

bootstrap_servers = "redpanda-0:9092"

With the IP and port allocated for redpanda. You might be interested in templating https://developer.hashicorp.com/nomad/docs/job-specification/template#consul-integration .